Skip to content

Commit 337ef77

Browse files
KSemenenkoKSemenenko
committed
Fix Azure upload metadata and refresh packages
1 parent bdcd7a2 commit 337ef77

9 files changed

Lines changed: 282 additions & 48 deletions

File tree

‎Directory.Build.props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
<RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
3030
<PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
3131
<Product>Managed Code - Storage</Product>
32-
<Version>10.0.6</Version>
33-
<PackageVersion>10.0.6</PackageVersion>
32+
<Version>10.0.7</Version>
33+
<PackageVersion>10.0.7</PackageVersion>
3434

3535
</PropertyGroup>
3636

‎Directory.Packages.props‎

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,51 +3,51 @@
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
44
</PropertyGroup>
55
<ItemGroup>
6-
<PackageVersion Include="AWSSDK.S3" Version="4.0.23" />
6+
<PackageVersion Include="AWSSDK.S3" Version="4.0.102.1" />
77
<PackageVersion Include="Azure.Identity" Version="1.21.0" />
8-
<PackageVersion Include="Azure.Storage.Blobs" Version="12.27.0" />
9-
<PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.25.0" />
10-
<PackageVersion Include="coverlet.collector" Version="10.0.0" />
11-
<PackageVersion Include="coverlet.msbuild" Version="10.0.0" />
8+
<PackageVersion Include="Azure.Storage.Blobs" Version="12.29.1" />
9+
<PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.27.1" />
10+
<PackageVersion Include="coverlet.collector" Version="10.0.1" />
11+
<PackageVersion Include="coverlet.msbuild" Version="10.0.1" />
1212
<PackageVersion Include="CsvHelper" Version="33.1.0" />
1313
<PackageVersion Include="DotNet.ReproducibleBuilds" Version="1.2.25" />
14-
<PackageVersion Include="Dropbox.Api" Version="7.0.0" />
14+
<PackageVersion Include="Dropbox.Api" Version="7.2.0" />
1515
<PackageVersion Include="Google.Api.Gax" Version="4.14.0" />
1616
<PackageVersion Include="Google.Api.Gax.Rest" Version="4.14.0" />
17-
<PackageVersion Include="Google.Apis.Drive.v3" Version="1.74.0.4135" />
18-
<PackageVersion Include="Google.Apis.Storage.v1" Version="1.73.0.4115" />
19-
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.14.0" />
17+
<PackageVersion Include="Google.Apis.Drive.v3" Version="1.75.0.4218" />
18+
<PackageVersion Include="Google.Apis.Storage.v1" Version="1.75.0.4193" />
19+
<PackageVersion Include="Google.Cloud.Storage.V1" Version="4.15.0" />
2020
<PackageVersion Include="Humanizer.Core" Version="3.0.10" />
21-
<PackageVersion Include="ManagedCode.Communication" Version="10.0.1" />
22-
<PackageVersion Include="ManagedCode.MimeTypes" Version="10.0.1" />
23-
<PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.3.9" />
24-
<PackageVersion Include="Microsoft.AspNetCore.Http.Connections.Client" Version="10.0.7" />
25-
<PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.7" />
26-
<PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="10.0.7" />
27-
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.7" />
28-
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.7" />
29-
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.7" />
30-
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.7" />
31-
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.7" />
32-
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.7" />
33-
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.7" />
34-
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.7" />
35-
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.7" />
36-
<PackageVersion Include="Microsoft.Graph" Version="5.105.0" />
37-
<PackageVersion Include="Microsoft.JSInterop" Version="10.0.7" />
38-
<PackageVersion Include="Microsoft.Playwright" Version="1.59.0" />
39-
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.5.1" />
40-
<PackageVersion Include="Microsoft.Orleans.Runtime" Version="10.1.0" />
41-
<PackageVersion Include="Microsoft.Orleans.TestingHost" Version="10.1.0" />
21+
<PackageVersion Include="ManagedCode.Communication" Version="10.1.1" />
22+
<PackageVersion Include="ManagedCode.MimeTypes" Version="10.0.10" />
23+
<PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.3.12" />
24+
<PackageVersion Include="Microsoft.AspNetCore.Http.Connections.Client" Version="10.0.11" />
25+
<PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.11" />
26+
<PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="10.0.11" />
27+
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.11" />
28+
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.11" />
29+
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.11" />
30+
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.11" />
31+
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.11" />
32+
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.11" />
33+
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.11" />
34+
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.11" />
35+
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.11" />
36+
<PackageVersion Include="Microsoft.Graph" Version="6.5.0" />
37+
<PackageVersion Include="Microsoft.JSInterop" Version="10.0.11" />
38+
<PackageVersion Include="Microsoft.Playwright" Version="1.62.0" />
39+
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.9.0" />
40+
<PackageVersion Include="Microsoft.Orleans.Runtime" Version="10.2.2" />
41+
<PackageVersion Include="Microsoft.Orleans.TestingHost" Version="10.2.2" />
4242
<PackageVersion Include="NetArchTest.Rules" Version="1.3.2" />
4343
<PackageVersion Include="Shouldly" Version="4.3.0" />
44-
<PackageVersion Include="SSH.NET" Version="2025.1.0" />
45-
<PackageVersion Include="Testcontainers" Version="4.11.0" />
46-
<PackageVersion Include="Testcontainers.Azurite" Version="4.11.0" />
47-
<PackageVersion Include="Testcontainers.FakeGcsServer" Version="4.11.0" />
48-
<PackageVersion Include="Testcontainers.LocalStack" Version="4.11.0" />
49-
<PackageVersion Include="Testcontainers.Sftp" Version="4.11.0" />
44+
<PackageVersion Include="SSH.NET" Version="2026.0.0" />
45+
<PackageVersion Include="Testcontainers" Version="4.14.0" />
46+
<PackageVersion Include="Testcontainers.Azurite" Version="4.14.0" />
47+
<PackageVersion Include="Testcontainers.FakeGcsServer" Version="4.14.0" />
48+
<PackageVersion Include="Testcontainers.LocalStack" Version="4.14.0" />
49+
<PackageVersion Include="Testcontainers.Sftp" Version="4.14.0" />
5050
<PackageVersion Include="xunit" Version="2.9.3" />
51-
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
51+
<PackageVersion Include="xunit.runner.visualstudio" Version="4.0.0" />
5252
</ItemGroup>
53-
</Project>
53+
</Project>

‎Storages/ManagedCode.Storage.Azure/AzureStorage.cs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,14 +241,18 @@ protected override async Task<Result<BlobMetadata>> UploadInternalAsync(Stream s
241241
CancellationToken cancellationToken = default)
242242
{
243243
var blobClient = StorageClient.GetBlobClient(options.FullPath);
244+
var contentLength = stream.CanSeek
245+
? checked((ulong)(stream.Length - stream.Position))
246+
: 0;
244247

245248
var uploadOptions = new BlobUploadOptions
246249
{
247250
Metadata = options.Metadata,
248251
HttpHeaders = new BlobHttpHeaders
249252
{
250253
ContentType = options.MimeType
251-
}
254+
},
255+
ProgressHandler = stream.CanSeek ? null : new UploadLengthProgress()
252256
};
253257

254258
uploadOptions.TransferOptions = ResolveUploadTransferOptions();
@@ -259,10 +263,23 @@ protected override async Task<Result<BlobMetadata>> UploadInternalAsync(Stream s
259263
cancellationToken.ThrowIfCancellationRequested();
260264
var blobInfo = await blobClient.UploadAsync(stream, uploadOptions, cancellationToken);
261265
cancellationToken.ThrowIfCancellationRequested();
262-
var metadataOptions = MetadataOptions.FromBaseOptions(options);
263-
metadataOptions.ETag = blobInfo.Value?.ETag.ToString() ?? string.Empty;
266+
contentLength = stream.CanSeek
267+
? contentLength
268+
: ((UploadLengthProgress)uploadOptions.ProgressHandler!).BytesTransferred;
269+
var lastModified = blobInfo.Value.LastModified;
264270

265-
return await GetBlobMetadataInternalAsync(metadataOptions, cancellationToken);
271+
return Result<BlobMetadata>.Succeed(new BlobMetadata
272+
{
273+
FullName = blobClient.Name,
274+
Name = Path.GetFileName(blobClient.Name),
275+
Uri = blobClient.Uri,
276+
Container = blobClient.BlobContainerName,
277+
Length = contentLength,
278+
CreatedOn = lastModified,
279+
LastModified = lastModified,
280+
Metadata = options.Metadata?.ToDictionary(pair => pair.Key, pair => pair.Value),
281+
MimeType = options.MimeType
282+
});
266283
}
267284
catch (Exception ex)
268285
{

‎Storages/ManagedCode.Storage.Azure/UploadLengthProgress.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
using System;
2+
using System.Threading;
3+
4+
namespace ManagedCode.Storage.Azure;
5+
6+
internal sealed class UploadLengthProgress : IProgress<long>
7+
{
8+
private long _bytesTransferred;
9+
10+
public ulong BytesTransferred => checked((ulong)Volatile.Read(ref _bytesTransferred));
11+
12+
public void Report(long value)
13+
{
14+
ArgumentOutOfRangeException.ThrowIfNegative(value);
15+
16+
var current = Volatile.Read(ref _bytesTransferred);
17+
while (value > current)
18+
{
19+
var observed = Interlocked.CompareExchange(ref _bytesTransferred, value, current);
20+
if (observed == current)
21+
return;
22+
23+
current = observed;
24+
}
25+
}
26+
}

‎Tests/ManagedCode.Storage.Tests/Common/EmptyContainer.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using DotNet.Testcontainers.Configurations;
77
using DotNet.Testcontainers.Containers;
88
using DotNet.Testcontainers.Images;
9+
using DotNet.Testcontainers.Networks;
910
using Microsoft.Extensions.Logging;
1011
using Microsoft.Extensions.Logging.Abstractions;
1112

@@ -79,6 +80,16 @@ public Task UnpauseAsync(CancellationToken ct = default)
7980
return Task.FromException(new NotImplementedException());
8081
}
8182

83+
public Task ConnectAsync(string network, CancellationToken ct = default)
84+
{
85+
return Task.CompletedTask;
86+
}
87+
88+
public Task ConnectAsync(INetwork network, CancellationToken ct = default)
89+
{
90+
return Task.CompletedTask;
91+
}
92+
8293
public async Task CopyAsync(byte[] fileContent, string filePath, uint uid = 0, uint gid = 0,
8394
UnixFileModes fileMode = UnixFileModes.None | UnixFileModes.OtherRead | UnixFileModes.GroupRead | UnixFileModes.UserWrite | UnixFileModes.UserRead, CancellationToken ct = new CancellationToken())
8495
{

‎Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureUploadTests.cs‎

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,27 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Text;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using Azure.Core;
8+
using Azure.Core.Pipeline;
9+
using Azure.Storage.Blobs;
10+
using ManagedCode.Storage.Azure.Extensions;
11+
using ManagedCode.Storage.Core.Models;
112
using ManagedCode.Storage.Tests.Common;
213
using ManagedCode.Storage.Tests.Storages.Abstracts;
314
using Microsoft.Extensions.DependencyInjection;
15+
using Shouldly;
416
using Testcontainers.Azurite;
17+
using Xunit;
518

619
namespace ManagedCode.Storage.Tests.Storages.Azure;
720

821
public class AzureUploadTests : UploadTests<AzuriteContainer>
922
{
23+
private readonly UploadRequestPolicy _requestPolicy = new();
24+
1025
protected override AzuriteContainer Build()
1126
{
1227
return new AzuriteBuilder(ContainerImages.Azurite)
@@ -16,6 +31,111 @@ protected override AzuriteContainer Build()
1631

1732
protected override ServiceProvider ConfigureServices()
1833
{
19-
return AzureConfigurator.ConfigureServices(Container.GetConnectionString());
34+
var clientOptions = new BlobClientOptions();
35+
clientOptions.AddPolicy(_requestPolicy, HttpPipelinePosition.PerCall);
36+
37+
var services = new ServiceCollection();
38+
services.AddAzureStorageAsDefault(options =>
39+
{
40+
options.Container = "managed-code-bucket";
41+
options.ConnectionString = Container.GetConnectionString();
42+
options.OriginalOptions = clientOptions;
43+
});
44+
return services.BuildServiceProvider();
45+
}
46+
47+
[Theory]
48+
[InlineData(true)]
49+
[InlineData(false)]
50+
public async Task UploadAsync_ReturnsCompleteMetadata_WithoutReadingBlobProperties(bool seekable)
51+
{
52+
var createResult = await Storage.CreateContainerAsync();
53+
createResult.IsSuccess.ShouldBeTrue();
54+
_requestPolicy.Reset();
55+
56+
var content = Encoding.UTF8.GetBytes("upload response metadata");
57+
await using Stream source = seekable
58+
? new MemoryStream(content)
59+
: new NonSeekableReadStream(new MemoryStream(content));
60+
var options = new UploadOptions
61+
{
62+
FileName = "metadata.txt",
63+
Directory = "uploads",
64+
MimeType = "text/plain",
65+
Metadata = new Dictionary<string, string> { ["purpose"] = "regression" }
66+
};
67+
68+
var result = await Storage.UploadAsync(source, options);
69+
70+
result.IsSuccess.ShouldBeTrue();
71+
result.Value.ShouldNotBeNull();
72+
result.Value!.FullName.ShouldBe("uploads/metadata.txt");
73+
result.Value.Name.ShouldBe("metadata.txt");
74+
result.Value.Container.ShouldBe("managed-code-bucket");
75+
result.Value.Length.ShouldBe((ulong)content.Length);
76+
result.Value.MimeType.ShouldBe("text/plain");
77+
result.Value.Metadata.ShouldNotBeNull();
78+
result.Value.Metadata!["purpose"].ShouldBe("regression");
79+
result.Value.CreatedOn.ShouldBe(result.Value.LastModified);
80+
_requestPolicy.BlobHeadRequestCount.ShouldBe(0);
81+
_requestPolicy.BlobWriteRequestCount.ShouldBeGreaterThan(0);
82+
}
83+
84+
private sealed class UploadRequestPolicy : HttpPipelineSynchronousPolicy
85+
{
86+
private int _blobHeadRequestCount;
87+
private int _blobWriteRequestCount;
88+
89+
public int BlobHeadRequestCount => Volatile.Read(ref _blobHeadRequestCount);
90+
public int BlobWriteRequestCount => Volatile.Read(ref _blobWriteRequestCount);
91+
92+
public void Reset()
93+
{
94+
Volatile.Write(ref _blobHeadRequestCount, 0);
95+
Volatile.Write(ref _blobWriteRequestCount, 0);
96+
}
97+
98+
public override void OnSendingRequest(HttpMessage message)
99+
{
100+
if (!message.Request.Uri.Path.EndsWith("/metadata.txt", StringComparison.Ordinal))
101+
return;
102+
103+
if (message.Request.Method == RequestMethod.Head)
104+
Interlocked.Increment(ref _blobHeadRequestCount);
105+
else if (message.Request.Method == RequestMethod.Put)
106+
Interlocked.Increment(ref _blobWriteRequestCount);
107+
}
108+
}
109+
110+
private sealed class NonSeekableReadStream(Stream inner) : Stream
111+
{
112+
public override bool CanRead => inner.CanRead;
113+
public override bool CanSeek => false;
114+
public override bool CanWrite => false;
115+
public override long Length => throw new NotSupportedException();
116+
117+
public override long Position
118+
{
119+
get => throw new NotSupportedException();
120+
set => throw new NotSupportedException();
121+
}
122+
123+
public override void Flush() => inner.Flush();
124+
public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count);
125+
public override int Read(Span<byte> buffer) => inner.Read(buffer);
126+
public override ValueTask<int> ReadAsync(
127+
Memory<byte> buffer,
128+
CancellationToken cancellationToken = default) => inner.ReadAsync(buffer, cancellationToken);
129+
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
130+
public override void SetLength(long value) => throw new NotSupportedException();
131+
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
132+
133+
protected override void Dispose(bool disposing)
134+
{
135+
if (disposing)
136+
inner.Dispose();
137+
138+
base.Dispose(disposing);
139+
}
20140
}
21141
}

‎Tests/ManagedCode.Storage.Tests/Storages/Orleans/ManagedCodeOrleansClusterFixture.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,14 +175,14 @@ private async Task EnsureContainersCreatedAsync()
175175

176176
internal static class ManagedCodeOrleansClusterTestMatrix
177177
{
178-
public static readonly TheoryData<string> AllProviders =
179-
[
178+
public static readonly TheoryData<string> AllProviders = new()
179+
{
180180
ManagedCodeOrleansProviderNames.FileSystem,
181181
ManagedCodeOrleansProviderNames.Azure,
182182
ManagedCodeOrleansProviderNames.Aws,
183183
ManagedCodeOrleansProviderNames.Gcp,
184184
ManagedCodeOrleansProviderNames.Sftp
185-
];
185+
};
186186
}
187187

188188
internal sealed record ManagedCodeOrleansClusterSettingsSnapshot(

0 commit comments

Comments
 (0)