Skip to content

Commit 63e0614

Browse files
KSemenenkoclaude
KSemenenko
and
claude
committed
add expected SHA-256 to verified writes and release 10.1.10
WriteIfAbsentOrSameAsync and WriteBytesIfAbsentOrSameAsync get overloads that take the caller's hexadecimal SHA-256. The staged or in-memory content is compared before commit, and a mismatch throws StorageUploadDigestException without leaving an object behind. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 21dc462 commit 63e0614

7 files changed

Lines changed: 228 additions & 10 deletions

File tree

‎Directory.Build.props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
<RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
3030
<PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
3131
<Product>Managed Code - Storage</Product>
32-
<Version>10.1.9</Version>
33-
<PackageVersion>10.1.9</PackageVersion>
32+
<Version>10.1.10</Version>
33+
<PackageVersion>10.1.10</PackageVersion>
3434

3535
</PropertyGroup>
3636

‎ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.Bytes.cs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,21 @@ namespace ManagedCode.Storage.Core.Primitives;
99
public static partial class VerifiedObjectUpload
1010
{
1111
/// <summary>Atomically writes a small immutable object and accepts only an exact existing retry.</summary>
12-
public static async Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsync(
12+
public static Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsync(
1313
this IObjectStorage storage,
1414
string path,
1515
ReadOnlyMemory<byte> content,
1616
StorageWriteOptions? options = null,
17-
CancellationToken cancellationToken = default)
17+
CancellationToken cancellationToken = default) =>
18+
WriteBytesVerifiedAsync(storage, path, content, null, options, cancellationToken);
19+
20+
private static async Task<VerifiedObjectUploadResult> WriteBytesVerifiedAsync(
21+
IObjectStorage storage,
22+
string path,
23+
ReadOnlyMemory<byte> content,
24+
byte[]? expectedSha256,
25+
StorageWriteOptions? options,
26+
CancellationToken cancellationToken)
1827
{
1928
ArgumentNullException.ThrowIfNull(storage);
2029
options ??= new StorageWriteOptions();
@@ -23,7 +32,8 @@ public static async Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsy
2332
throw new ArgumentException("An immutable write cannot use IfMatch.", nameof(options));
2433
}
2534

26-
var expectedHash = SHA256.HashData(content.Span);
35+
var contentHash = SHA256.HashData(content.Span);
36+
ThrowIfDigestDiffers(contentHash, expectedSha256);
2737
StorageObjectInfo info;
2838
var reused = false;
2939
try
@@ -41,7 +51,7 @@ public static async Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsy
4151
!string.Equals(info.ContentEncoding, options.ContentEncoding, StringComparison.OrdinalIgnoreCase)) ||
4252
(options.Metadata is not null && !MetadataMatches(options.Metadata, info.Metadata)) ||
4353
!await MatchesExistingAsync(storage, path, info.ETag, content.Length,
44-
expectedHash, cancellationToken).ConfigureAwait(false))
54+
contentHash, cancellationToken).ConfigureAwait(false))
4555
{
4656
throw;
4757
}
@@ -54,6 +64,6 @@ public static async Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsy
5464
throw new StorageUploadLengthException(info.Length > content.Length);
5565
}
5666

57-
return new VerifiedObjectUploadResult(info, Convert.ToHexStringLower(expectedHash), reused);
67+
return new VerifiedObjectUploadResult(info, Convert.ToHexStringLower(contentHash), reused);
5868
}
5969
}

‎ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.ExpectedDigest.cs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
using System;
2+
using System.Buffers;
3+
using System.IO;
4+
using System.Security.Cryptography;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
8+
namespace ManagedCode.Storage.Core.Primitives;
9+
10+
/// <summary>The uploaded content does not match the SHA-256 its caller declared; nothing was committed.</summary>
11+
public sealed class StorageUploadDigestException()
12+
: IOException("Upload content does not match its declared SHA-256.");
13+
14+
public static partial class VerifiedObjectUpload
15+
{
16+
private const int Sha256ByteLength = 32;
17+
private const int Sha256HexLength = Sha256ByteLength * 2;
18+
19+
/// <summary>
20+
/// Stages and verifies an immutable upload against its declared length and hexadecimal SHA-256. A digest mismatch
21+
/// throws <see cref="StorageUploadDigestException"/> before commit; identical retries reuse the existing revision.
22+
/// </summary>
23+
public static async Task<VerifiedObjectUploadResult> WriteIfAbsentOrSameAsync(
24+
this IObjectStorage storage,
25+
string path,
26+
Stream content,
27+
long expectedLength,
28+
string expectedSha256,
29+
StorageWriteOptions? options = null,
30+
CancellationToken cancellationToken = default) =>
31+
await WriteVerifiedAsync(storage, path, content, expectedLength, ParseSha256(expectedSha256), options,
32+
cancellationToken).ConfigureAwait(false);
33+
34+
/// <summary>
35+
/// Atomically writes a small immutable object whose content must match the hexadecimal SHA-256. A digest mismatch
36+
/// throws <see cref="StorageUploadDigestException"/> before anything is written.
37+
/// </summary>
38+
public static async Task<VerifiedObjectUploadResult> WriteBytesIfAbsentOrSameAsync(
39+
this IObjectStorage storage,
40+
string path,
41+
ReadOnlyMemory<byte> content,
42+
string expectedSha256,
43+
StorageWriteOptions? options = null,
44+
CancellationToken cancellationToken = default) =>
45+
await WriteBytesVerifiedAsync(storage, path, content, ParseSha256(expectedSha256), options,
46+
cancellationToken).ConfigureAwait(false);
47+
48+
private static byte[] ParseSha256(string expectedSha256)
49+
{
50+
ArgumentNullException.ThrowIfNull(expectedSha256);
51+
var digest = new byte[Sha256ByteLength];
52+
if (expectedSha256.Length != Sha256HexLength ||
53+
Convert.FromHexString(expectedSha256, digest, out _, out _) != OperationStatus.Done)
54+
{
55+
throw new ArgumentException("The expected SHA-256 must be 64 hexadecimal characters.",
56+
nameof(expectedSha256));
57+
}
58+
59+
return digest;
60+
}
61+
62+
private static void ThrowIfDigestDiffers(byte[] actual, byte[]? expected)
63+
{
64+
if (expected is not null && !CryptographicOperations.FixedTimeEquals(actual, expected))
65+
{
66+
throw new StorageUploadDigestException();
67+
}
68+
}
69+
}

‎ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,23 @@ public static partial class VerifiedObjectUpload
2020
private const int PartBytes = 4 * 1024 * 1024;
2121

2222
/// <summary>Stages and verifies an immutable upload before atomic commit. Identical retries reuse the existing revision.</summary>
23-
public static async Task<VerifiedObjectUploadResult> WriteIfAbsentOrSameAsync(
23+
public static Task<VerifiedObjectUploadResult> WriteIfAbsentOrSameAsync(
2424
this IObjectStorage storage,
2525
string path,
2626
Stream content,
2727
long expectedLength,
2828
StorageWriteOptions? options = null,
29-
CancellationToken cancellationToken = default)
29+
CancellationToken cancellationToken = default) =>
30+
WriteVerifiedAsync(storage, path, content, expectedLength, null, options, cancellationToken);
31+
32+
private static async Task<VerifiedObjectUploadResult> WriteVerifiedAsync(
33+
IObjectStorage storage,
34+
string path,
35+
Stream content,
36+
long expectedLength,
37+
byte[]? expectedSha256,
38+
StorageWriteOptions? options,
39+
CancellationToken cancellationToken)
3040
{
3141
ArgumentNullException.ThrowIfNull(storage);
3242
ArgumentNullException.ThrowIfNull(content);
@@ -44,6 +54,7 @@ public static async Task<VerifiedObjectUploadResult> WriteIfAbsentOrSameAsync(
4454

4555
var (partIds, inputHash) = await StageAsync(multipart, path, content, expectedLength,
4656
cancellationToken).ConfigureAwait(false);
57+
ThrowIfDigestDiffers(inputHash, expectedSha256);
4758
StorageObjectInfo info;
4859
var reused = false;
4960
try

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,8 @@ Immutable uploads can call `WriteIfAbsentOrSameAsync` on `IObjectStorage` with a
481481

482482
Payloads already in memory can use the `ReadOnlyMemory<byte>` overloads of `WriteObjectAsync` and `StagePartAsync` (and `WriteBytesIfAbsentOrSameAsync` for immutable writes). Array-backed memory, including slices, is written in place without copying.
483483

484+
When the caller already knows the content digest, the `WriteIfAbsentOrSameAsync` and `WriteBytesIfAbsentOrSameAsync` overloads with an `expectedSha256` (64 hexadecimal characters) verify it before committing. A mismatch throws `StorageUploadDigestException` and leaves no object behind.
485+
484486
### ASP.NET Streaming Controllers
485487

486488
Controllers in `ManagedCode.Storage.Server` expose minimal routes that stream directly between HTTP clients and blob providers. Uploads arrive as multipart forms or raw streams, flow through the unified `IStorage` abstraction, and land in whichever provider is registered. Downloads return `FileStreamResult` responses so browsers, SDKs, or background jobs can read blobs without buffering the whole payload in memory.

‎Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureVerifiedUploadDigestTests.cs‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
using System;
2+
using System.IO;
3+
using System.Security.Cryptography;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
using ManagedCode.Storage.Azure;
7+
using ManagedCode.Storage.Azure.Options;
8+
using ManagedCode.Storage.Core.Primitives;
9+
using ManagedCode.Storage.Tests.Common;
10+
using Shouldly;
11+
using Testcontainers.Azurite;
12+
using Xunit;
13+
14+
namespace ManagedCode.Storage.Tests.Storages.Azure;
15+
16+
public sealed class AzureVerifiedUploadDigestTests : IAsyncLifetime
17+
{
18+
private const int MultipartPayloadBytes = 4 * 1024 * 1024 + 9;
19+
20+
private readonly AzuriteContainer _container = new AzuriteBuilder(ContainerImages.Azurite)
21+
.WithCommand("--skipApiVersionCheck").Build();
22+
23+
public ValueTask InitializeAsync() => new(_container.StartAsync());
24+
public ValueTask DisposeAsync() => _container.DisposeAsync();
25+
26+
[Fact]
27+
public async Task StreamUpload_WithMatchingDigest_CommitsAndReusesRetry()
28+
{
29+
var cancellationToken = TestContext.Current.CancellationToken;
30+
using var storage = CreateStorage();
31+
var objects = await CreateContainerAsync(storage);
32+
var bytes = RandomNumberGenerator.GetBytes(MultipartPayloadBytes);
33+
var digest = Convert.ToHexString(SHA256.HashData(bytes));
34+
35+
using var first = new MemoryStream(bytes, writable: false);
36+
var written = await objects.WriteIfAbsentOrSameAsync("archive.zip", first, bytes.LongLength, digest,
37+
cancellationToken: cancellationToken);
38+
using var retry = new MemoryStream(bytes, writable: false);
39+
var replay = await objects.WriteIfAbsentOrSameAsync("archive.zip", retry, bytes.LongLength,
40+
digest.ToLowerInvariant(), cancellationToken: cancellationToken);
41+
42+
written.ReusedExisting.ShouldBeFalse();
43+
written.Sha256.ShouldBe(digest.ToLowerInvariant());
44+
written.Info.Length.ShouldBe(bytes.LongLength);
45+
replay.ReusedExisting.ShouldBeTrue();
46+
replay.Info.ETag.ShouldBe(written.Info.ETag);
47+
}
48+
49+
[Fact]
50+
public async Task StreamUpload_WithDifferentDigest_CommitsNothing()
51+
{
52+
var cancellationToken = TestContext.Current.CancellationToken;
53+
using var storage = CreateStorage();
54+
var objects = await CreateContainerAsync(storage);
55+
var bytes = Encoding.UTF8.GetBytes("declared content");
56+
var otherDigest = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes("other content")));
57+
58+
using var source = new MemoryStream(bytes, writable: false);
59+
await Should.ThrowAsync<StorageUploadDigestException>(() => objects.WriteIfAbsentOrSameAsync(
60+
"mismatch.bin", source, bytes.LongLength, otherDigest, cancellationToken: cancellationToken));
61+
62+
(await objects.ObjectExistsAsync("mismatch.bin", cancellationToken)).ShouldBeFalse();
63+
}
64+
65+
[Fact]
66+
public async Task ByteUpload_WithExpectedDigest_WritesOnlyMatchingContent()
67+
{
68+
var cancellationToken = TestContext.Current.CancellationToken;
69+
using var storage = CreateStorage();
70+
var objects = await CreateContainerAsync(storage);
71+
var bytes = Encoding.UTF8.GetBytes("{\"value\":1}");
72+
var digest = Convert.ToHexStringLower(SHA256.HashData(bytes));
73+
var options = new StorageWriteOptions { ContentType = "application/json" };
74+
75+
await Should.ThrowAsync<StorageUploadDigestException>(() => objects.WriteBytesIfAbsentOrSameAsync(
76+
"payload.json", Encoding.UTF8.GetBytes("{\"value\":2}"), digest, options, cancellationToken));
77+
(await objects.ObjectExistsAsync("payload.json", cancellationToken)).ShouldBeFalse();
78+
79+
var written = await objects.WriteBytesIfAbsentOrSameAsync("payload.json", bytes, digest, options,
80+
cancellationToken);
81+
var replay = await objects.WriteBytesIfAbsentOrSameAsync("payload.json", bytes, digest, options,
82+
cancellationToken);
83+
84+
written.ReusedExisting.ShouldBeFalse();
85+
written.Sha256.ShouldBe(digest);
86+
replay.ReusedExisting.ShouldBeTrue();
87+
replay.Info.ETag.ShouldBe(written.Info.ETag);
88+
}
89+
90+
[Theory]
91+
[InlineData("abc")]
92+
[InlineData("zz00000000000000000000000000000000000000000000000000000000000000")]
93+
public async Task Upload_WithMalformedDigest_RejectsBeforeWriting(string expectedSha256)
94+
{
95+
var cancellationToken = TestContext.Current.CancellationToken;
96+
using var storage = CreateStorage();
97+
var objects = await CreateContainerAsync(storage);
98+
var bytes = Encoding.UTF8.GetBytes("content");
99+
100+
using var source = new MemoryStream(bytes, writable: false);
101+
(await Should.ThrowAsync<ArgumentException>(() => objects.WriteIfAbsentOrSameAsync(
102+
"malformed.bin", source, bytes.LongLength, expectedSha256, cancellationToken: cancellationToken)))
103+
.ParamName.ShouldBe("expectedSha256");
104+
(await Should.ThrowAsync<ArgumentException>(() => objects.WriteBytesIfAbsentOrSameAsync(
105+
"malformed.bin", bytes, expectedSha256, cancellationToken: cancellationToken)))
106+
.ParamName.ShouldBe("expectedSha256");
107+
108+
(await objects.ObjectExistsAsync("malformed.bin", cancellationToken)).ShouldBeFalse();
109+
}
110+
111+
private AzureStorage CreateStorage() => new(new AzureStorageOptions
112+
{
113+
ConnectionString = _container.GetConnectionString(),
114+
Container = $"verified-digest-{Guid.NewGuid():N}",
115+
CreateContainerIfNotExists = false
116+
});
117+
118+
private static async Task<IObjectStorage> CreateContainerAsync(AzureStorage storage)
119+
{
120+
var objects = storage.RequireObjectStorage();
121+
await objects.CreatePrivateContainerAsync(cancellationToken: TestContext.Current.CancellationToken);
122+
return objects;
123+
}
124+
}

‎docs/Architecture.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,9 @@ bounded, file-backed SHA-256 read and `VerifiedObjectUpload` for immutable
6666
stream or byte writes with exact retry comparison, including reconciliation
6767
after a write commits but its response is lost. `ObjectStorageBytes` adds in-memory
6868
`WriteObjectAsync` and `StagePartAsync` overloads that write array-backed
69-
memory in place, copying only memory without a backing array.
69+
memory in place, copying only memory without a backing array. Verified writes
70+
accept an optional expected SHA-256 and throw `StorageUploadDigestException`
71+
before commit when the staged content differs.
7072

7173
## Cartograph artifact provider
7274

0 commit comments

Comments
 (0)