Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Add an implementation of ReadAsync to ZipAESStream #576

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

[WIP] Add an implementation of ReadAsync to ZipAESStream #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9da6d27
add an extra unit test for doing an async read on a ZipFile InputStream
Numpsy Feb 14, 2021
08f8186
Add an async version of ReadRequestedBytes
Numpsy Feb 14, 2021
2423484
start adding an implementation of ReadAsync to ZipAESStream
Numpsy Feb 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 50 additions & 10 deletions src/ICSharpCode.SharpZipLib/Core/StreamUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.IO;
using System.Threading.Tasks;

namespace ICSharpCode.SharpZipLib.Core
{
Expand Down Expand Up @@ -64,16 +65,8 @@ static public void ReadFully(Stream stream, byte[] buffer, int offset, int count
}
}

/// <summary>
/// Read as much data as possible from a <see cref="Stream"/>", up to the requested number of bytes
/// </summary>
/// <param name="stream">The stream to read data from.</param>
/// <param name="buffer">The buffer to store data in.</param>
/// <param name="offset">The offset at which to begin storing data.</param>
/// <param name="count">The number of bytes of data to store.</param>
/// <exception cref="ArgumentNullException">Required parameter is null</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="offset"/> and or <paramref name="count"/> are invalid.</exception>
static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, int count)
// A helper function to share between the async and sync versions of ReadRequestedBytes
private static void ValidateArgumentsForRead(Stream stream, byte[] buffer, int offset, int count)
{
if (stream == null)
{
Expand All @@ -95,7 +88,23 @@ static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, i
{
throw new ArgumentOutOfRangeException(nameof(count));
}
}

/// <summary>
/// Read as much data as possible from a <see cref="Stream"/>", up to the requested number of bytes
/// </summary>
/// <param name="stream">The stream to read data from.</param>
/// <param name="buffer">The buffer to store data in.</param>
/// <param name="offset">The offset at which to begin storing data.</param>
/// <param name="count">The number of bytes of data to store.</param>
/// <exception cref="ArgumentNullException">Required parameter is null</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="offset"/> and or <paramref name="count"/> are invalid.</exception>
static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, int count)
{
// Common validation function
ValidateArgumentsForRead(stream, buffer, offset, count);

// read the data using Read
int totalReadCount = 0;
while (count > 0)
{
Expand All @@ -112,6 +121,37 @@ static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, i
return totalReadCount;
}

/// <summary>
/// Read as much data as possible from a <see cref="Stream"/>", up to the requested number of bytes
/// </summary>
/// <param name="stream">The stream to read data from.</param>
/// <param name="buffer">The buffer to store data in.</param>
/// <param name="offset">The offset at which to begin storing data.</param>
/// <param name="count">The number of bytes of data to store.</param>
/// <exception cref="ArgumentNullException">Required parameter is null</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="offset"/> and or <paramref name="count"/> are invalid.</exception>
static public async Task<int> ReadRequestedBytesAsync(Stream stream, byte[] buffer, int offset, int count)
{
// Common validation function
ValidateArgumentsForRead(stream, buffer, offset, count);

// read the data using ReadAsync
int totalReadCount = 0;
while (count > 0)
{
int readCount = await stream.ReadAsync(buffer, offset, count);
if (readCount <= 0)
{
break;
}
offset += readCount;
count -= readCount;
totalReadCount += readCount;
}

return totalReadCount;
}

/// <summary>
/// Copy the contents of one <see cref="Stream"/> to another.
/// </summary>
Expand Down
185 changes: 130 additions & 55 deletions src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using ICSharpCode.SharpZipLib.Core;

namespace ICSharpCode.SharpZipLib.Encryption
Expand Down Expand Up @@ -70,18 +72,11 @@ public override int Read(byte[] buffer, int offset, int count)
return 0;

// If we have buffered data, read that first
int nBytes = 0;
if (HasBufferedData)
{
nBytes = ReadBufferedData(buffer, offset, count);
int nBytes = ReadBufferedData(buffer, ref offset, ref count);

// Read all requested data from the buffer
if (nBytes == count)
return nBytes;

offset += nBytes;
count -= nBytes;
}
// Read all requested data from the buffer
if (nBytes == count)
return nBytes;

// Read more data from the input, if available
if (_slideBuffer != null)
Expand All @@ -90,6 +85,27 @@ public override int Read(byte[] buffer, int offset, int count)
return nBytes;
}

/// <inheritdoc/>
public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
// Nothing to do
if (count == 0)
return 0;

// If we have buffered data, read that first
int nBytes = ReadBufferedData(buffer, ref offset, ref count);

// Read all requested data from the buffer
if (nBytes == count)
return nBytes;

// Read more data from the input, if available
if (_slideBuffer != null)
nBytes += await ReadAndTransformAsync(buffer, offset, count);

return nBytes;
}

// Read data from the underlying stream and decrypt it
private int ReadAndTransform(byte[] buffer, int offset, int count)
{
Expand All @@ -105,68 +121,127 @@ private int ReadAndTransform(byte[] buffer, int offset, int count)
// Maintain a read-ahead equal to the length of (crypto block + Auth Code).
// When that runs out we can detect these final sections.
int lengthToRead = BLOCK_AND_AUTH - byteCount;
if (_slideBuffer.Length - _slideBufFreePos < lengthToRead)
UpdateSlideBufferIfNeeded(lengthToRead);

int obtained = StreamUtils.ReadRequestedBytes(_stream, _slideBuffer, _slideBufFreePos, lengthToRead);
_slideBufFreePos += obtained;

// Transform data from the slide buffer
if (TransformFromSlideBuffer(buffer, ref offset, bytesLeftToRead, ref nBytes))
{
// Shift the data to the beginning of the buffer
int iTo = 0;
for (int iFrom = _slideBufStartPos; iFrom < _slideBufFreePos; iFrom++, iTo++)
{
_slideBuffer[iTo] = _slideBuffer[iFrom];
}
_slideBufFreePos -= _slideBufStartPos; // Note the -=
_slideBufStartPos = 0;
// Reached the auth code
break;
}
int obtained = StreamUtils.ReadRequestedBytes(_stream, _slideBuffer, _slideBufFreePos, lengthToRead);
}
return nBytes;
}

// Read data from the underlying stream asynchronously and decrypt it
private async Task<int> ReadAndTransformAsync(byte[] buffer, int offset, int count)
{
int nBytes = 0;
while (nBytes < count)
{
int bytesLeftToRead = count - nBytes;

// Calculate buffer quantities vs read-ahead size, and check for sufficient free space
int byteCount = _slideBufFreePos - _slideBufStartPos;

// Need to handle final block and Auth Code specially, but don't know total data length.
// Maintain a read-ahead equal to the length of (crypto block + Auth Code).
// When that runs out we can detect these final sections.
int lengthToRead = BLOCK_AND_AUTH - byteCount;
UpdateSlideBufferIfNeeded(lengthToRead);

int obtained = await StreamUtils.ReadRequestedBytesAsync(_stream, _slideBuffer, _slideBufFreePos, lengthToRead);
_slideBufFreePos += obtained;

// Recalculate how much data we now have
byteCount = _slideBufFreePos - _slideBufStartPos;
if (byteCount >= BLOCK_AND_AUTH)
// Transform data from the slide buffer
if (TransformFromSlideBuffer(buffer, ref offset, bytesLeftToRead, ref nBytes))
{
var read = TransformAndBufferBlock(buffer, offset, bytesLeftToRead, CRYPTO_BLOCK_SIZE);
nBytes += read;
offset += read;
// Reached the auth code
break;
}
else
}
return nBytes;
}

// Helper function to update the slide buffer, if we need to
private void UpdateSlideBufferIfNeeded(int lengthToRead)
{
if (_slideBuffer.Length - _slideBufFreePos < lengthToRead)
{
// Shift the data to the beginning of the buffer
int iTo = 0;
for (int iFrom = _slideBufStartPos; iFrom < _slideBufFreePos; iFrom++, iTo++)
{
// Last round.
if (byteCount > AUTH_CODE_LENGTH)
{
// At least one byte of data plus auth code
int finalBlock = byteCount - AUTH_CODE_LENGTH;
nBytes += TransformAndBufferBlock(buffer, offset, bytesLeftToRead, finalBlock);
}
else if (byteCount < AUTH_CODE_LENGTH)
throw new Exception("Internal error missed auth code"); // Coding bug
// Final block done. Check Auth code.
byte[] calcAuthCode = _transform.GetAuthCode();
for (int i = 0; i < AUTH_CODE_LENGTH; i++)
_slideBuffer[iTo] = _slideBuffer[iFrom];
}
_slideBufFreePos -= _slideBufStartPos; // Note the -=
_slideBufStartPos = 0;
}
}

// A helper to do the non-async crypto transform, using data from the in-memory slide buffer
// Returns true if the auth code has been reached, false if not.
private bool TransformFromSlideBuffer(byte[] buffer, ref int offset, int bytesLeftToRead, ref int nBytes)
{
// Recalculate how much data we now have
int byteCount = _slideBufFreePos - _slideBufStartPos;
if (byteCount >= BLOCK_AND_AUTH)
{
var read = TransformAndBufferBlock(buffer, offset, bytesLeftToRead, CRYPTO_BLOCK_SIZE);
nBytes += read;
offset += read;

return false;
}
else
{
// Last round.
if (byteCount > AUTH_CODE_LENGTH)
{
// At least one byte of data plus auth code
int finalBlock = byteCount - AUTH_CODE_LENGTH;
nBytes += TransformAndBufferBlock(buffer, offset, bytesLeftToRead, finalBlock);
}
else if (byteCount < AUTH_CODE_LENGTH)
throw new Exception("Internal error missed auth code"); // Coding bug
// Final block done. Check Auth code.
byte[] calcAuthCode = _transform.GetAuthCode();
for (int i = 0; i < AUTH_CODE_LENGTH; i++)
{
if (calcAuthCode[i] != _slideBuffer[_slideBufStartPos + i])
{
if (calcAuthCode[i] != _slideBuffer[_slideBufStartPos + i])
{
throw new Exception("AES Authentication Code does not match. This is a super-CRC check on the data in the file after compression and encryption. \r\n"
Numpsy marked this conversation as resolved.
Show resolved Hide resolved
+ "The file may be damaged.");
}
throw new Exception("AES Authentication Code does not match. This is a super-CRC check on the data in the file after compression and encryption. \r\n"
+ "The file may be damaged.");
}
}

// don't need this any more, so use it as a 'complete' flag
_slideBuffer = null;
// don't need this any more, so use it as a 'complete' flag
_slideBuffer = null;

break; // Reached the auth code
}
return true; // Reached the auth code
}
return nBytes;
}

// read some buffered data
private int ReadBufferedData(byte[] buffer, int offset, int count)
private int ReadBufferedData(byte[] buffer, ref int offset, ref int count)
{
int copyCount = Math.Min(count, _transformBufferFreePos - _transformBufferStartPos);
if (HasBufferedData)
{
int copyCount = Math.Min(count, _transformBufferFreePos - _transformBufferStartPos);

Array.Copy(_transformBuffer, _transformBufferStartPos, buffer, offset, copyCount);
_transformBufferStartPos += copyCount;

Array.Copy(_transformBuffer, _transformBufferStartPos, buffer, offset, copyCount);
_transformBufferStartPos += copyCount;
offset += copyCount;
count -= copyCount;

return copyCount;
}

return copyCount;
return 0;
}

// Perform the crypto transform, and buffer the data if less than one block has been requested.
Expand Down
48 changes: 48 additions & 0 deletions test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.IO;
using System.Text;
using ICSharpCode.SharpZipLib.Tests.TestSupport;
using System.Threading.Tasks;

namespace ICSharpCode.SharpZipLib.Tests.Zip
{
Expand Down Expand Up @@ -180,6 +181,53 @@ public void ZipFileStoreAes()
}
}

/// <summary>
/// As <see cref="ZipFileStoreAes"/>, but with Async reads
/// </summary>
[Test]
[Category("Encryption")]
[Category("Zip")]
public async Task ZipFileStoreAesAsync()
{
string password = "password";

using (var memoryStream = new MemoryStream())
{
// Try to create a zip stream
WriteEncryptedZipToStream(memoryStream, password, 256, CompressionMethod.Stored);

// reset
memoryStream.Seek(0, SeekOrigin.Begin);

// try to read it
var zipFile = new ZipFile(memoryStream, leaveOpen: true)
{
Password = password
};

foreach (ZipEntry entry in zipFile)
{
if (!entry.IsFile) continue;

// Should be stored rather than deflated
Assert.That(entry.CompressionMethod, Is.EqualTo(CompressionMethod.Stored), "Entry should be stored");

using (var zis = zipFile.GetInputStream(entry))
{
var buffer = new byte[entry.Size];

using (var inputStream = zipFile.GetInputStream(entry))
{
await zis.ReadAsync(buffer, 0, buffer.Length);
}

var content = Encoding.UTF8.GetString(buffer);
Assert.That(content, Is.EqualTo(DummyDataString), "Decompressed content does not match input data");
}
}
}
}

/// <summary>
/// Test using AES encryption on a file whose contents are Stored rather than deflated
/// </summary>
Expand Down