Skip to content

Commit b40903f

Browse files
carlossanlopcarlossanlop
authored
Zip async implementation (#114421)
Added new public async APIs for System.IO.Compression and System.IO.Compression.ZipFile. Modified existing unit tests to validate both sync and async depending on an async argument. Added fuzzing tests for ZipArchive. Added new tests to confirm that no sync API is calling async APIs internally and viceversa.
1 parent b281500 commit b40903f

File tree

58 files changed

+7396
-2620
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+7396
-2620
lines changed

‎eng/pipelines/libraries/fuzzing/deploy-to-onefuzz.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,4 +169,12 @@ extends:
169169
onefuzzDropDirectory: $(fuzzerProject)/deployment/Utf8JsonWriterFuzzer
170170
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
171171
displayName: Send Utf8JsonWriterFuzzer to OneFuzz
172+
173+
- task: onefuzz-task@0
174+
inputs:
175+
onefuzzOSes: 'Windows'
176+
env:
177+
onefuzzDropDirectory: $(fuzzerProject)/deployment/ZipArchiveFuzzer
178+
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
179+
displayName: Send ZipArchiveFuzzer to OneFuzz
172180
# ONEFUZZ_TASK_WORKAROUND_END

‎src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public async Task FlushAsync_DuringFlushAsync()
107107
{
108108
byte[] buffer = null;
109109
string testFilePath = CompressedTestFile(UncompressedTestFile());
110-
using (var origStream = await LocalMemoryStream.readAppFileAsync(testFilePath))
110+
using (var origStream = await LocalMemoryStream.ReadAppFileAsync(testFilePath))
111111
{
112112
buffer = origStream.ToArray();
113113
}
@@ -164,8 +164,8 @@ public virtual async Task Dispose_WithUnfinishedReadAsync()
164164
[MemberData(nameof(UncompressedTestFiles))]
165165
public async Task Read(string testFile)
166166
{
167-
var uncompressedStream = await LocalMemoryStream.readAppFileAsync(testFile);
168-
var compressedStream = await LocalMemoryStream.readAppFileAsync(CompressedTestFile(testFile));
167+
var uncompressedStream = await LocalMemoryStream.ReadAppFileAsync(testFile);
168+
var compressedStream = await LocalMemoryStream.ReadAppFileAsync(CompressedTestFile(testFile));
169169
using var decompressor = CreateStream(compressedStream, CompressionMode.Decompress);
170170
var decompressorOutput = new MemoryStream();
171171

@@ -199,7 +199,7 @@ public async Task Read(string testFile)
199199
[Fact]
200200
public async Task Read_EndOfStreamPosition()
201201
{
202-
var compressedStream = await LocalMemoryStream.readAppFileAsync(CompressedTestFile(UncompressedTestFile()));
202+
var compressedStream = await LocalMemoryStream.ReadAppFileAsync(CompressedTestFile(UncompressedTestFile()));
203203
int compressedEndPosition = (int)compressedStream.Length;
204204
var rand = new Random(1024);
205205
int _bufferSize = BufferSize * 2 - 568;
@@ -219,7 +219,7 @@ public async Task Read_EndOfStreamPosition()
219219
public async Task Read_BaseStreamSlowly()
220220
{
221221
string testFile = UncompressedTestFile();
222-
var uncompressedStream = await LocalMemoryStream.readAppFileAsync(testFile);
222+
var uncompressedStream = await LocalMemoryStream.ReadAppFileAsync(testFile);
223223
var compressedStream = new BadWrappedStream(BadWrappedStream.Mode.ReadSlowly, File.ReadAllBytes(CompressedTestFile(testFile)));
224224
using var decompressor = CreateStream(compressedStream, CompressionMode.Decompress);
225225
var decompressorOutput = new MemoryStream();
@@ -354,7 +354,7 @@ public async Task TestLeaveOpenAfterValidDecompress()
354354
//Create the Stream
355355
int _bufferSize = 1024;
356356
var bytes = new byte[_bufferSize];
357-
Stream compressedStream = await LocalMemoryStream.readAppFileAsync(CompressedTestFile(UncompressedTestFile()));
357+
Stream compressedStream = await LocalMemoryStream.ReadAppFileAsync(CompressedTestFile(UncompressedTestFile()));
358358
Stream decompressor = CreateStream(compressedStream, CompressionMode.Decompress, leaveOpen: false);
359359

360360
//Read some data and Close the stream
@@ -426,7 +426,7 @@ public void BaseStreamTest(CompressionMode mode)
426426
[InlineData(CompressionMode.Decompress)]
427427
public async Task BaseStream_Modify(CompressionMode mode)
428428
{
429-
using (var baseStream = await LocalMemoryStream.readAppFileAsync(CompressedTestFile(UncompressedTestFile())))
429+
using (var baseStream = await LocalMemoryStream.ReadAppFileAsync(CompressedTestFile(UncompressedTestFile())))
430430
using (var compressor = CreateStream(baseStream, mode))
431431
{
432432
int size = 1024;
@@ -457,7 +457,7 @@ public void BaseStream_NullAfterDisposeWithFalseLeaveOpen(CompressionMode mode)
457457
[InlineData(CompressionMode.Decompress)]
458458
public async Task BaseStream_ValidAfterDisposeWithTrueLeaveOpen(CompressionMode mode)
459459
{
460-
var ms = await LocalMemoryStream.readAppFileAsync(CompressedTestFile(UncompressedTestFile()));
460+
var ms = await LocalMemoryStream.ReadAppFileAsync(CompressedTestFile(UncompressedTestFile()));
461461
using var decompressor = CreateStream(ms, mode, leaveOpen: true);
462462
var baseStream = BaseStream(decompressor);
463463
Assert.Same(ms, baseStream);
@@ -475,7 +475,7 @@ public async Task BaseStream_ValidAfterDisposeWithTrueLeaveOpen(CompressionMode
475475
[MemberData(nameof(UncompressedTestFilesZLib))]
476476
public async Task CompressionLevel_SizeInOrder(string testFile)
477477
{
478-
using var uncompressedStream = await LocalMemoryStream.readAppFileAsync(testFile);
478+
using var uncompressedStream = await LocalMemoryStream.ReadAppFileAsync(testFile);
479479

480480
async Task<long> GetLengthAsync(CompressionLevel compressionLevel)
481481
{
@@ -501,7 +501,7 @@ async Task<long> GetLengthAsync(CompressionLevel compressionLevel)
501501
[MemberData(nameof(UncompressedTestFilesZLib))]
502502
public async Task ZLibCompressionOptions_SizeInOrder(string testFile)
503503
{
504-
using var uncompressedStream = await LocalMemoryStream.readAppFileAsync(testFile);
504+
using var uncompressedStream = await LocalMemoryStream.ReadAppFileAsync(testFile);
505505

506506
async Task<long> GetLengthAsync(int compressionLevel)
507507
{
@@ -512,7 +512,7 @@ async Task<long> GetLengthAsync(int compressionLevel)
512512
await compressor.FlushAsync();
513513
return mms.Length;
514514
}
515-
515+
516516
long fastestLength = await GetLengthAsync(1);
517517
long optimalLength = await GetLengthAsync(5);
518518
long smallestLength = await GetLengthAsync(9);
@@ -525,7 +525,7 @@ async Task<long> GetLengthAsync(int compressionLevel)
525525
[MemberData(nameof(ZLibOptionsRoundTripTestData))]
526526
public async Task RoundTripWithZLibCompressionOptions(string testFile, ZLibCompressionOptions options)
527527
{
528-
using var uncompressedStream = await LocalMemoryStream.readAppFileAsync(testFile);
528+
using var uncompressedStream = await LocalMemoryStream.ReadAppFileAsync(testFile);
529529
var compressedStream = await CompressTestFile(uncompressedStream, options);
530530
using var decompressor = CreateStream(compressedStream, mode: CompressionMode.Decompress);
531531
using var decompressorOutput = new MemoryStream();

‎src/libraries/Common/tests/System/IO/Compression/LocalMemoryStream.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public LocalMemoryStream Clone()
1616
return local;
1717
}
1818

19-
public static async Task<LocalMemoryStream> readAppFileAsync(string testFile)
19+
public static async Task<LocalMemoryStream> ReadAppFileAsync(string testFile)
2020
{
2121
var baseStream = await StreamHelpers.CreateTempCopyStream(testFile);
2222
var ms = new LocalMemoryStream();

‎src/libraries/Common/tests/System/IO/Compression/NoAsyncCallsStream.cs

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
namespace System.IO.Compression.Tests;
8+
9+
// A stream meant to be used for testing that an implementation's sync methods do not accidentally call any async methods.
10+
internal sealed class NoAsyncCallsStream : Stream
11+
{
12+
private readonly Stream _s;
13+
14+
public NoAsyncCallsStream(Stream stream) => _s = stream;
15+
16+
// Allows temporarily disabling the current stream's sync API usage restriction.
17+
public bool IsRestrictionEnabled { get; set; }
18+
19+
public override bool CanRead => _s.CanRead;
20+
public override bool CanSeek => _s.CanSeek;
21+
public override bool CanTimeout => _s.CanTimeout;
22+
public override bool CanWrite => _s.CanWrite;
23+
public override long Length => _s.Length;
24+
public override long Position { get => _s.Position; set => _s.Position = value; }
25+
public override int ReadTimeout { get => _s.ReadTimeout; set => _s.ReadTimeout = value; }
26+
public override int WriteTimeout { get => _s.WriteTimeout; set => _s.WriteTimeout = value; }
27+
public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => _s.BeginRead(buffer, offset, count, callback, state);
28+
public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => _s.BeginWrite(buffer, offset, count, callback, state);
29+
public override void Close() => _s.Close();
30+
public override int EndRead(IAsyncResult asyncResult) => _s.EndRead(asyncResult);
31+
public override void EndWrite(IAsyncResult asyncResult) => _s.EndWrite(asyncResult);
32+
public override bool Equals(object? obj) => _s.Equals(obj);
33+
public override int GetHashCode() => _s.GetHashCode();
34+
public override int ReadByte() => _s.ReadByte();
35+
public override long Seek(long offset, SeekOrigin origin) => _s.Seek(offset, origin);
36+
public override void SetLength(long value) => _s.SetLength(value);
37+
public override string? ToString() => _s.ToString();
38+
39+
// Sync
40+
public override void CopyTo(Stream destination, int bufferSize) => _s.CopyTo(destination, bufferSize);
41+
protected override void Dispose(bool disposing) => _s.Dispose();
42+
public override void Flush() => _s.Flush();
43+
public override int Read(byte[] buffer, int offset, int count) => _s.Read(buffer, offset, count);
44+
public override int Read(Span<byte> buffer) => _s.Read(buffer);
45+
public override void Write(byte[] buffer, int offset, int count) => _s.Write(buffer, offset, count);
46+
public override void Write(ReadOnlySpan<byte> buffer) => _s.Write(buffer);
47+
public override void WriteByte(byte value) => _s.WriteByte(value);
48+
49+
// Async
50+
public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) =>
51+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.CopyToAsync(destination, bufferSize, cancellationToken);
52+
public override ValueTask DisposeAsync() =>
53+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.DisposeAsync();
54+
public override Task FlushAsync(CancellationToken cancellationToken) =>
55+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.FlushAsync();
56+
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) =>
57+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.ReadAsync(buffer, offset, count, cancellationToken);
58+
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>
59+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.ReadAsync(buffer, cancellationToken);
60+
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) =>
61+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.WriteAsync(buffer, offset, count, cancellationToken);
62+
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default) =>
63+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.WriteAsync(buffer, cancellationToken);
64+
}

‎src/libraries/Common/tests/System/IO/Compression/NoSyncCallsStream.cs

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
namespace System.IO.Compression.Tests;
8+
9+
// A stream meant to be used for testing that an implementation's async methods do not accidentally call any sync methods.
10+
internal sealed class NoSyncCallsStream : Stream
11+
{
12+
private readonly Stream _s;
13+
14+
public NoSyncCallsStream(Stream stream) => _s = stream;
15+
16+
// Allows temporarily disabling the current stream's sync API usage restriction.
17+
public bool IsRestrictionEnabled { get; set; }
18+
19+
public override bool CanRead => _s.CanRead;
20+
public override bool CanSeek => _s.CanSeek;
21+
public override bool CanTimeout => _s.CanTimeout;
22+
public override bool CanWrite => _s.CanWrite;
23+
public override long Length => _s.Length;
24+
public override long Position { get => _s.Position; set => _s.Position = value; }
25+
public override int ReadTimeout { get => _s.ReadTimeout; set => _s.ReadTimeout = value; }
26+
public override int WriteTimeout { get => _s.WriteTimeout; set => _s.WriteTimeout = value; }
27+
public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => _s.BeginRead(buffer, offset, count, callback, state);
28+
public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) => _s.BeginWrite(buffer, offset, count, callback, state);
29+
public override void Close() => _s.Close();
30+
public override int EndRead(IAsyncResult asyncResult) => _s.EndRead(asyncResult);
31+
public override void EndWrite(IAsyncResult asyncResult) => _s.EndWrite(asyncResult);
32+
public override bool Equals(object? obj) => _s.Equals(obj);
33+
public override int GetHashCode() => _s.GetHashCode();
34+
public override int ReadByte() => _s.ReadByte();
35+
public override long Seek(long offset, SeekOrigin origin) => _s.Seek(offset, origin);
36+
public override void SetLength(long value) => _s.SetLength(value);
37+
public override string? ToString() => _s.ToString();
38+
39+
// Sync
40+
public override void CopyTo(Stream destination, int bufferSize)
41+
{
42+
if (IsRestrictionEnabled)
43+
{
44+
throw new InvalidOperationException();
45+
}
46+
else
47+
{
48+
_s.CopyTo(destination, bufferSize);
49+
}
50+
}
51+
protected override void Dispose(bool disposing)
52+
{
53+
// _disposing = true;
54+
if (IsRestrictionEnabled)
55+
{
56+
throw new InvalidOperationException();
57+
}
58+
else
59+
{
60+
_s.Dispose();
61+
}
62+
}
63+
public override void Flush()
64+
{
65+
if (IsRestrictionEnabled)
66+
{
67+
throw new InvalidOperationException();
68+
}
69+
else
70+
{
71+
_s.Flush();
72+
}
73+
}
74+
public override int Read(byte[] buffer, int offset, int count) =>
75+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.Read(buffer, offset, count);
76+
public override int Read(Span<byte> buffer) =>
77+
IsRestrictionEnabled ? throw new InvalidOperationException() : _s.Read(buffer);
78+
public override void Write(byte[] buffer, int offset, int count)
79+
{
80+
bool isDeflateStream = false;
81+
82+
// Get the stack trace to determine the calling method
83+
var stackTrace = new System.Diagnostics.StackTrace();
84+
var callingMethod = stackTrace.GetFrame(1)?.GetMethod();
85+
86+
// Check if the calling method belongs to the DeflateStream class
87+
if (callingMethod?.DeclaringType == typeof(System.IO.Compression.DeflateStream))
88+
{
89+
isDeflateStream = true;
90+
}
91+
92+
if (!isDeflateStream && IsRestrictionEnabled)
93+
{
94+
throw new InvalidOperationException($"Parent class is {callingMethod.DeclaringType}");
95+
}
96+
else
97+
{
98+
_s.Write(buffer, offset, count);
99+
}
100+
}
101+
public override void Write(ReadOnlySpan<byte> buffer)
102+
{
103+
if (IsRestrictionEnabled)
104+
{
105+
throw new InvalidOperationException();
106+
}
107+
else
108+
{
109+
_s.Write(buffer);
110+
}
111+
}
112+
public override void WriteByte(byte value)
113+
{
114+
if (IsRestrictionEnabled)
115+
{
116+
throw new InvalidOperationException();
117+
}
118+
else
119+
{
120+
_s.WriteByte(value);
121+
}
122+
}
123+
124+
// Async
125+
public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) => _s.CopyToAsync(destination, bufferSize, cancellationToken);
126+
public override ValueTask DisposeAsync() => _s.DisposeAsync();
127+
public override Task FlushAsync(CancellationToken cancellationToken) => _s.FlushAsync(cancellationToken);
128+
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _s.ReadAsync(buffer, offset, count, cancellationToken);
129+
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) => _s.ReadAsync(buffer, cancellationToken);
130+
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _s.WriteAsync(buffer, offset, count, cancellationToken);
131+
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default) => _s.WriteAsync(buffer, cancellationToken);
132+
}

‎src/libraries/Common/tests/System/IO/Compression/StreamHelpers.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ public static partial class StreamHelpers
99
{
1010
public static async Task<MemoryStream> CreateTempCopyStream(string path)
1111
{
12-
var bytes = File.ReadAllBytes(path);
12+
var bytes = await File.ReadAllBytesAsync(path);
1313

1414
var ms = new MemoryStream();
1515
await ms.WriteAsync(bytes, 0, bytes.Length);

0 commit comments

Comments
 (0)