Skip to content

Commit eb92f8c

Browse files
committed
add Cartograph storage provider
1 parent 454e446 commit eb92f8c

29 files changed

Lines changed: 1534 additions & 1 deletion

‎AGENTS.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ This file defines how AI agents work in this solution.
2626
- `Integraions/ManagedCode.Storage.Server/`
2727
- `Storages/ManagedCode.Storage.Aws/`
2828
- `Storages/ManagedCode.Storage.Browser/`
29+
- `Storages/ManagedCode.Storage.Cartograph/`
2930
- `Storages/ManagedCode.Storage.Azure/`
3031
- `Storages/ManagedCode.Storage.Azure.DataLake/`
3132
- `Storages/ManagedCode.Storage.CloudKit/`
@@ -47,6 +48,13 @@ This file defines how AI agents work in this solution.
4748
4. Local `AGENTS.md` files may refine or tighten root rules, but they must not silently weaken them.
4849
5. If a local rule needs an exception, document it explicitly in the nearest local `AGENTS.md`, ADR, or feature doc.
4950

51+
## ManagedCode Dependency Ownership And Repair
52+
53+
- `ManagedCode.*` projects and packages are maintained by us. Locate a defective dependency's owning repository alongside this checkout, read its `AGENTS.md`, and repair it there instead of replacing it, copying its implementation, or hiding the defect in a consumer workaround.
54+
- Deliver dependency repairs with focused regression tests, required repository checks, a canonical patch-version increment, a scoped commit, and a push to GitHub. These scoped version, commit, push, and release-trigger actions have standing user authorization; preserve unrelated work and never force-push or bypass protections.
55+
- Follow the owning repository's GitHub release workflow to success and verify the new version on the intended NuGet feed before updating a consumer reference and running consumer regressions. Diagnose publication failures in the owning repository; a pushed commit or green build alone is not delivery.
56+
- Do not claim delivery with unpublished local packages or temporary project references. Report the exact source, credentials, permissions, or release-infrastructure blocker when delivery cannot proceed.
57+
5058
## Conversations (Self-Learning)
5159

5260
Learn the user's stable habits, preferences, and corrections. Record durable rules here instead of relying on chat history.
@@ -291,6 +299,8 @@ Toolchain notes:
291299

292300
### Git And PRs
293301

302+
- Never run `git stash` or enable automatic stashing; keep working-tree changes visible so concurrent work stays reviewable.
303+
- When the user requests committing all current changes, commit the full requested working-tree scope on the currently checked-out branch; do not silently omit existing changes or move them to another branch.
294304
- Write commit subjects in the imperative mood such as `add ftp retry policy` and keep them provider-scoped.
295305
- Group related edits in one commit and avoid WIP spam.
296306
- PRs should summarize impact, list touched projects, reference issues, and note new configuration or secrets.

‎Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
<PackageVersion Include="Azure.Identity" Version="1.21.0" />
88
<PackageVersion Include="Azure.Storage.Blobs" Version="12.29.2" />
99
<PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.27.1" />
10+
<PackageVersion Include="Cartograph.Catalog" Version="0.1.0-alpha" />
1011
<PackageVersion Include="coverlet.collector" Version="10.0.1" />
1112
<PackageVersion Include="coverlet.msbuild" Version="10.0.1" />
1213
<PackageVersion Include="CsvHelper" Version="33.1.0" />

‎ManagedCode.Storage.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<Folder Name="/Storages/">
1414
<Project Path="Storages/ManagedCode.Storage.Aws/ManagedCode.Storage.Aws.csproj" />
1515
<Project Path="Storages/ManagedCode.Storage.Browser/ManagedCode.Storage.Browser.csproj" />
16+
<Project Path="Storages/ManagedCode.Storage.Cartograph/ManagedCode.Storage.Cartograph.csproj" />
1617
<Project Path="Storages/ManagedCode.Storage.Azure.DataLake/ManagedCode.Storage.Azure.DataLake.csproj" />
1718
<Project Path="Storages/ManagedCode.Storage.Azure/ManagedCode.Storage.Azure.csproj" />
1819
<Project Path="Storages/ManagedCode.Storage.FileSystem/ManagedCode.Storage.FileSystem.csproj" />

‎README.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ Cross-provider blob storage toolkit for .NET and ASP.NET streaming scenarios.
3333
- [Motivation](#motivation)
3434
- [Features](#features)
3535
- [Packages](#packages)
36+
- [Cartograph artifacts](#cartograph-artifacts)
3637
- [Architecture](#architecture)
3738
- [Virtual File System (VFS)](#virtual-file-system-vfs)
3839
- [Dependency Injection & Keyed Registrations](#dependency-injection--keyed-registrations)
@@ -131,13 +132,46 @@ Cloud storage vendors expose distinct SDKs, option models, and authentication pa
131132
| [ManagedCode.Storage.Aws](https://www.nuget.org/packages/ManagedCode.Storage.Aws) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Aws.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Aws) | Amazon S3 provider with Object Lock and legal hold operations. |
132133
| [ManagedCode.Storage.Gcp](https://www.nuget.org/packages/ManagedCode.Storage.Gcp) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Gcp.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Gcp) | Google Cloud Storage integration built on official SDKs. |
133134
| [ManagedCode.Storage.Browser](https://www.nuget.org/packages/ManagedCode.Storage.Browser) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Browser.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Browser) | Browser storage provider with Blazor DI helpers, MVC/static-asset helpers, IndexedDB metadata, and OPFS-backed payload streaming. |
135+
| [ManagedCode.Storage.Cartograph](https://www.nuget.org/packages/ManagedCode.Storage.Cartograph) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Cartograph.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Cartograph) | Read-only provider for files inside immutable Cartograph artifacts, with mapped streaming, catalog metadata, and directory listings. |
134136
| [ManagedCode.Storage.FileSystem](https://www.nuget.org/packages/ManagedCode.Storage.FileSystem) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.FileSystem.svg)](https://www.nuget.org/packages/ManagedCode.Storage.FileSystem) | Local file system implementation for hybrid or on-premises workloads. |
135137
| [ManagedCode.Storage.Sftp](https://www.nuget.org/packages/ManagedCode.Storage.Sftp) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Sftp.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Sftp) | SFTP provider powered by SSH.NET for regulated and air-gapped environments. |
136138
| [ManagedCode.Storage.OneDrive](https://www.nuget.org/packages/ManagedCode.Storage.OneDrive) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.OneDrive.svg)](https://www.nuget.org/packages/ManagedCode.Storage.OneDrive) | OneDrive provider built on Microsoft Graph. |
137139
| [ManagedCode.Storage.GoogleDrive](https://www.nuget.org/packages/ManagedCode.Storage.GoogleDrive) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.GoogleDrive.svg)](https://www.nuget.org/packages/ManagedCode.Storage.GoogleDrive) | Google Drive provider built on the Google Drive API. |
138140
| [ManagedCode.Storage.Dropbox](https://www.nuget.org/packages/ManagedCode.Storage.Dropbox) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.Dropbox.svg)](https://www.nuget.org/packages/ManagedCode.Storage.Dropbox) | Dropbox provider built on the Dropbox API. |
139141
| [ManagedCode.Storage.CloudKit](https://www.nuget.org/packages/ManagedCode.Storage.CloudKit) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.Storage.CloudKit.svg)](https://www.nuget.org/packages/ManagedCode.Storage.CloudKit) | CloudKit (iCloud app data) provider built on CloudKit Web Services. |
140142

143+
### Cartograph artifacts
144+
145+
`ManagedCode.Storage.Cartograph` exposes files inside a [Cartograph](https://github.com/angelhernandezm/Cartograph) `.ctg` artifact through `IStorage`. Reads use Cartograph's mapped records and checksums without extracting or buffering the whole file.
146+
147+
```bash
148+
dotnet add package ManagedCode.Storage.Cartograph
149+
```
150+
151+
```csharp
152+
using ManagedCode.Storage.Cartograph.Extensions;
153+
using ManagedCode.Storage.Core;
154+
using Microsoft.Extensions.DependencyInjection;
155+
156+
var services = new ServiceCollection();
157+
services.AddCartographStorageAsDefault(options =>
158+
options.ArtifactPath = Path.GetFullPath("documents.ctg"));
159+
160+
await using var provider = services.BuildServiceProvider();
161+
var storage = provider.GetRequiredService<IStorage>();
162+
var result = await storage.GetStreamAsync("reports/annual.pdf");
163+
if (result.IsFailed)
164+
throw new IOException(result.Problem?.Detail);
165+
166+
await using var content = result.Value;
167+
await using var destination = File.Create("annual.pdf");
168+
await content.CopyToAsync(destination);
169+
```
170+
171+
Use `AddCartographStorage` for `ICartographStorage`, keyed overloads for multiple artifacts, or `AddStorageFactory` with `CreateCartographStorage` to open artifacts dynamically. Metadata, recursive directory listings, existence checks, and `DownloadAsync` use the same Storage APIs. Returned streams are sequential, read-only, and own their artifact until disposed.
172+
173+
The artifact must already contain a Cartograph file catalog. `CreateContainerAsync` validates it; upload, delete, container removal, and legal-hold operations return unsupported results because artifacts are immutable. The module uses the published `Cartograph.Catalog` `0.1.0-alpha` package and requires a 64-bit process. Keep artifacts unchanged while their streams are open. See the [Cartograph provider guide](docs/Features/provider-cartograph.md) for catalog requirements and full wiring examples.
174+
141175
### Configuring OneDrive, Google Drive, Dropbox, and CloudKit
142176

143177
> iCloud Drive does not expose a public server-side file API. `ManagedCode.Storage.CloudKit` targets CloudKit Web Services (iCloud app data), not iCloud Drive.
@@ -418,6 +452,7 @@ flowchart LR
418452
GoogleDrive["Google Drive"]
419453
Dropbox["Dropbox"]
420454
CloudKit["CloudKit (iCloud app data)"]
455+
Cartograph["Cartograph artifacts (read-only)"]
421456
Fs["File System"]
422457
Sftp["SFTP"]
423458
end
@@ -433,6 +468,7 @@ flowchart LR
433468
Factories --> GoogleDrive
434469
Factories --> Dropbox
435470
Factories --> CloudKit
471+
Factories --> Cartograph
436472
Factories --> Fs
437473
Factories --> Sftp
438474
```
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
Project: ManagedCode.Storage.Cartograph
2+
Owned by: ManagedCode.Storage maintainers
3+
4+
Parent: `../../AGENTS.md`
5+
6+
## Purpose
7+
8+
- Exposes files catalogued in immutable Cartograph artifacts through the normal Storage abstraction.
9+
- Keeps the Cartograph dependency and mapped-memory lifetime rules isolated in a separate provider package.
10+
11+
## Entry Points
12+
13+
- `CartographStorage.cs`
14+
- `ICartographStorage.cs`
15+
- `CartographStorageProvider.cs`
16+
- `Options/CartographStorageOptions.cs`
17+
- `Extensions/ServiceCollectionExtensions.cs`
18+
- `Extensions/StorageFactoryExtensions.cs`
19+
- `CartographReadStream.cs`
20+
- `CartographDownload.cs`
21+
22+
## Boundaries
23+
24+
- In scope: local artifact reading, catalog validation, file metadata/listing/downloads, provider options, default/typed/keyed DI, and factory creation.
25+
- Out of scope: changing Core contracts, duplicating the Cartograph format/parser, modifying immutable artifacts, remote transports, or test fakes.
26+
- Protected areas: lease lifetime, untrusted catalog paths and record spans, cancellation, checksum verification, and DI alias identity.
27+
28+
## Project Commands
29+
30+
- `build`: `dotnet build ManagedCode.Storage.Cartograph.csproj`
31+
- `test`: `dotnet test ../../Tests/ManagedCode.Storage.Tests/ManagedCode.Storage.Tests.csproj --configuration Release --filter "FullyQualifiedName~Cartograph"`
32+
- `format`: `dotnet format ../../ManagedCode.Storage.slnx`
33+
- Framework and runner: xUnit v2 / VSTest; analyzer policy comes from the root `.editorconfig`.
34+
35+
## Applicable Skills
36+
37+
- `mcaf-dotnet`
38+
- `mcaf-testing`
39+
- `mcaf-dotnet-xunit`
40+
- `mcaf-architecture-overview`
41+
- `mcaf-documentation`
42+
43+
## Local Rules And Risks
44+
45+
- Use the centrally pinned published `Cartograph.Catalog` package and its transitive Format/substrate packages.
46+
- Returned streams own an independent artifact snapshot and dispose every record lease before its artifact.
47+
- Never extract whole files to temporary storage or load their payload into `MemoryStream` to implement reads.
48+
- Keep per-record checksums enabled. Validate catalog entry spans and lengths against record-directory metadata without reading file payloads.
49+
- Resolve catalog-relative names with ordinal comparison. Ignore `SourceRoot`; catalog paths must never become host filesystem paths.
50+
- Mutations, legal holds, and container deletion return explicit unsupported results. `CreateContainerAsync` validates an existing artifact without creating one.
51+
- Downloads stream to a sibling staging file and replace the destination only after successful checksum-verified reading; always clean up staging files and preserve an existing destination on failure.
52+
- Required documentation: `docs/Architecture.md`, `docs/Features/provider-cartograph.md`, and the README usage section.
53+
- Inherit root maintainability limits and exception policy.
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using Cartograph.Catalog;
6+
using Cartograph.Format;
7+
8+
namespace ManagedCode.Storage.Cartograph;
9+
10+
internal sealed class CartographArtifact : IDisposable
11+
{
12+
private readonly Dictionary<string, CatalogEntry> _entries = new(StringComparer.Ordinal);
13+
14+
private CartographArtifact(Artifact artifact, string path)
15+
{
16+
Artifact = artifact;
17+
Path = path;
18+
if (artifact.Segments.Count == 0 || artifact.Segments[0].RecordCount != 1)
19+
throw new InvalidDataException("The first segment must contain a single file catalog record.");
20+
21+
using var record = artifact.ReadRecord(0);
22+
Catalog = FileCatalog.Deserialize(record.Sequence);
23+
foreach (var entry in Catalog.Entries)
24+
{
25+
ValidateEntry(entry);
26+
if (!_entries.TryAdd(entry.RelativePath, entry))
27+
throw new InvalidDataException($"Duplicate catalog path '{entry.RelativePath}'.");
28+
}
29+
}
30+
31+
public Artifact Artifact { get; }
32+
public FileCatalog Catalog { get; }
33+
public string Path { get; }
34+
35+
public static CartographArtifact Open(string path)
36+
{
37+
var artifact = Artifact.Open(path);
38+
try
39+
{
40+
return new CartographArtifact(artifact, path);
41+
}
42+
catch
43+
{
44+
artifact.Dispose();
45+
throw;
46+
}
47+
}
48+
49+
public CatalogEntry? Find(string path) => _entries.GetValueOrDefault(CartographPath.Normalize(path));
50+
51+
public CatalogEntry RequireEntry(string path) => Find(path)
52+
?? throw new FileNotFoundException($"File '{path}' is not present in the artifact.", path);
53+
54+
public Stream OpenRead(string path)
55+
{
56+
var entry = RequireEntry(path);
57+
return new CartographReadStream(this, entry);
58+
}
59+
60+
private void ValidateEntry(CatalogEntry entry)
61+
{
62+
CartographPath.ValidateCatalogPath(entry.RelativePath);
63+
if (entry.Length < 0 || entry.SegmentIndex <= 0 || entry.SegmentIndex >= Artifact.Segments.Count ||
64+
entry.RecordIndex < 0 || entry.RecordCount <= 0 ||
65+
entry.LastWriteUtcTicks < DateTime.MinValue.Ticks || entry.LastWriteUtcTicks > DateTime.MaxValue.Ticks)
66+
{
67+
throw new InvalidDataException($"Invalid catalog entry '{entry.RelativePath}'.");
68+
}
69+
70+
var segment = Artifact.Segments[entry.SegmentIndex];
71+
if (entry.RecordIndex > segment.RecordCount || entry.RecordCount > segment.RecordCount - entry.RecordIndex)
72+
throw new InvalidDataException($"Catalog record span is out of bounds for '{entry.RelativePath}'.");
73+
74+
var globalStart = Artifact.Segments.Take(entry.SegmentIndex).Sum(item => (long)item.RecordCount);
75+
if (entry.GlobalIndex != globalStart + entry.RecordIndex)
76+
throw new InvalidDataException($"Catalog record indices disagree for '{entry.RelativePath}'.");
77+
78+
long length = 0;
79+
for (var index = 0; index < entry.RecordCount; index++)
80+
length = checked(length + segment.GetRecordLength(entry.RecordIndex + index));
81+
82+
if (length != entry.Length)
83+
throw new InvalidDataException($"Catalog length disagrees with records for '{entry.RelativePath}'.");
84+
}
85+
86+
public void Dispose() => Artifact.Dispose();
87+
}
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System;
2+
using System.IO;
3+
using System.Threading;
4+
using System.Threading.Tasks;
5+
6+
namespace ManagedCode.Storage.Cartograph;
7+
8+
internal static class CartographDownload
9+
{
10+
private const int BufferSize = 80 * 1024;
11+
12+
public static async Task CopyToAsync(Stream source, string destinationPath, CancellationToken cancellationToken)
13+
{
14+
var stagingPath = destinationPath + '.' + Guid.NewGuid().ToString("N") + ".download";
15+
try
16+
{
17+
await using (var destination = new FileStream(stagingPath, FileMode.CreateNew, FileAccess.Write,
18+
FileShare.None, BufferSize, FileOptions.Asynchronous))
19+
await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false);
20+
21+
cancellationToken.ThrowIfCancellationRequested();
22+
File.Move(stagingPath, destinationPath, overwrite: true);
23+
}
24+
finally
25+
{
26+
File.Delete(stagingPath);
27+
}
28+
}
29+
}
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
using System;
2+
using System.IO;
3+
using System.Linq;
4+
5+
namespace ManagedCode.Storage.Cartograph;
6+
7+
internal static class CartographPath
8+
{
9+
public static string Normalize(string path)
10+
{
11+
ArgumentException.ThrowIfNullOrWhiteSpace(path);
12+
var normalized = path.Replace('\\', '/');
13+
if (normalized.StartsWith('/') || normalized.Contains(':') || normalized.Contains('\0') ||
14+
normalized.Split('/').Any(part => part.Length == 0 || part is "." or ".."))
15+
{
16+
throw new ArgumentException("Use a relative catalog path without empty or traversal segments.", nameof(path));
17+
}
18+
19+
return normalized;
20+
}
21+
22+
public static string DirectoryPrefix(string? directory)
23+
{
24+
if (string.IsNullOrEmpty(directory))
25+
return string.Empty;
26+
27+
return Normalize(directory.TrimEnd('/', '\\')) + '/';
28+
}
29+
30+
public static void ValidateCatalogPath(string path)
31+
{
32+
if (Normalize(path) != path)
33+
throw new InvalidDataException("Catalog paths must use forward slashes.");
34+
}
35+
36+
public static string ResolveFileSystemPath(string path)
37+
{
38+
var fullPath = Path.GetFullPath(path);
39+
var root = Path.GetPathRoot(fullPath) ?? throw new IOException("The path has no filesystem root.");
40+
var resolved = root;
41+
foreach (var component in fullPath[root.Length..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries))
42+
{
43+
resolved = Path.Combine(resolved, component);
44+
FileSystemInfo info = System.IO.Directory.Exists(resolved) ? new DirectoryInfo(resolved) : new FileInfo(resolved);
45+
if (info.LinkTarget is not null)
46+
{
47+
var target = info.ResolveLinkTarget(returnFinalTarget: true) ?? throw new IOException("Cannot resolve symbolic link.");
48+
resolved = ResolveFileSystemPath(target.FullName);
49+
}
50+
}
51+
52+
return resolved;
53+
}
54+
}

0 commit comments

Comments
 (0)