From 1cffd6fc705ce01f2edc2a9c30723eb3318a6f4e Mon Sep 17 00:00:00 2001 From: Yiqing Wang Date: Mon, 20 Jul 2026 00:05:36 -0700 Subject: [PATCH] atelet: add successful RPC boundary tests --- cmd/atelet/main.go | 88 ++++++++----- cmd/atelet/main_test.go | 214 ++++++++++++++++++++++++++++---- cmd/atelet/oci.go | 16 ++- cmd/atelet/oci_test.go | 4 +- cmd/atelet/sandbox_assets.go | 12 +- cmd/atelet/volumes.go | 4 +- internal/ateompath/ateompath.go | 30 ++--- 7 files changed, 283 insertions(+), 85 deletions(-) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 4f1e02f63..fefb5ebec 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -74,7 +74,7 @@ var ( gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") - imageCacheDir = pflag.String("image-cache-dir", ateompath.ImageCacheDir, "Directory for the node-local OCI image layer cache. Must be on the volume shared with the ateom pods (the cached layers are their overlay lowerdirs), and on a disk sized for both capacity and IOPS: unpack throughput is gated by the volume's IOPS.") + imageCacheDir = pflag.String("image-cache-dir", ateompath.ImageCacheDir(), "Directory for the node-local OCI image layer cache. Must be on the volume shared with the ateom pods (the cached layers are their overlay lowerdirs), and on a disk sized for both capacity and IOPS: unpack throughput is gated by the volume's IOPS.") showVersion = pflag.Bool("version", false, "Print version and exit.") ) @@ -207,10 +207,26 @@ func main() { type AteomHerder struct { ateletpb.UnimplementedAteomHerderServer - ateomDialer *AteomDialer - imageCache *imagecache.Store + ateomDialer ateomClientDialer + imageCache imageEnsurer anonGCSClient ategcs.ObjectStorage gcsClient ategcs.ObjectStorage + paths pathMapper +} + +// pathMapper redirects ateom shared-filesystem paths for tests. A nil mapper +// leaves production paths unchanged. +type pathMapper func(string) string + +func (m pathMapper) mapPath(path string) string { + if m == nil { + return path + } + return m(path) +} + +type ateomClientDialer interface { + DialAteom(context.Context, string) (ateompb.AteomClient, error) } var _ ateletpb.AteomHerderServer = (*AteomHerder)(nil) @@ -218,10 +234,10 @@ var _ ateletpb.AteomHerderServer = (*AteomHerder)(nil) // NewService creates a new WorkersManagerService. func NewService( ctx context.Context, - ateomDialer *AteomDialer, + ateomDialer ateomClientDialer, anonGCSClient ategcs.ObjectStorage, gcsClient ategcs.ObjectStorage, - imageCache *imagecache.Store, + imageCache imageEnsurer, ) *AteomHerder { wms := &AteomHerder{ ateomDialer: ateomDialer, @@ -250,7 +266,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * return nil, err } - if err := resetActorDirs(actorUID); err != nil { + if err := resetActorDirs(s.paths, actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } @@ -267,7 +283,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * // Record the sandbox binaries this actor is running so a later Checkpoint // (whose request no longer carries the sandbox config) can re-fetch the same // version and pin it into the snapshot manifest. - if err := writeSandboxRecord(actorUID, sandboxRec); err != nil { + if err := writeSandboxRecord(s.paths, actorUID, sandboxRec); err != nil { return nil, fmt.Errorf("while recording sandbox assets: %w", err) } @@ -347,7 +363,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // version this actor was started with from the on-node record and re-fetch // it (a cache hit) so ateom can drive runsc, and so we can pin it into the // snapshot manifest below. - sandboxRec, err := readSandboxRecord(actorUID) + sandboxRec, err := readSandboxRecord(s.paths, actorUID) if err != nil { return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidSandboxAsset, ateerrors.ReasonTerminalFileSystemError) } @@ -356,7 +372,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidSandboxAsset, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL) } - checkpointDir := ateompath.CheckpointStateDir(actorUID) + checkpointDir := s.paths.mapPath(ateompath.CheckpointStateDir(actorUID)) client, err := s.dialAteom(ctx, req.GetTargetAteomUid()) if err != nil { @@ -406,7 +422,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe _ = s.unmountExternalVolumes(ctx, actorUID, req.GetSpec().GetVolumes()) // Note: we do not crash the actor if resetting the directory fails. - if err := resetActorDirs(actorUID); err != nil { + if err := resetActorDirs(s.paths, actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } @@ -424,7 +440,7 @@ func toAteomSnapshotScope(scope ateletpb.SnapshotScope) ateompb.SnapshotScope { } func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { - localCheckpointPath := filepath.Join(ateompath.LocalCheckpointsDir(req.GetActorUid()), req.GetLocalConfig().GetSnapshotPrefix()) + localCheckpointPath := filepath.Join(s.paths.mapPath(ateompath.LocalCheckpointsDir(req.GetActorUid())), req.GetLocalConfig().GetSnapshotPrefix()) if err := os.MkdirAll(localCheckpointPath, 0o700); err != nil { return fmt.Errorf("while creating local checkpoint directory: %w", err) } @@ -494,7 +510,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // Not crashing the actor, because terminal errors here indicate problems with atelet, // node or the disk itself. - if err := resetActorDirs(actorUID); err != nil { + if err := resetActorDirs(s.paths, actorUID); err != nil { return nil, fmt.Errorf("while resetting actor dirs: %w", err) } @@ -508,7 +524,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) } }() - checkpointDir := ateompath.RestoreStateDir(actorUID) + checkpointDir := s.paths.mapPath(ateompath.RestoreStateDir(actorUID)) // Per-step timing so we can attribute resume latency between the rustfs // download/decompress, the OCI image unpack, and ateom's own work. Logged at the end. @@ -531,7 +547,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return nil, ateerrors.CrashIfReason(ctx, fmt.Errorf("while unmarshalling sandbox record: %w", err), ateerrors.ReasonInvalidSandboxAsset) } case ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL: - localCheckpointDir := ateompath.LocalCheckpointsDir(actorUID) + localCheckpointDir := s.paths.mapPath(ateompath.LocalCheckpointsDir(actorUID)) snapshotPrefix := req.GetLocalConfig().GetSnapshotPrefix() manifest, err := os.ReadFile(filepath.Join(localCheckpointDir, snapshotPrefix, sandboxManifestName)) if err != nil { @@ -564,7 +580,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError) } case ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL: - if err := s.copyLocalCheckpoint(gctx, req.GetLocalConfig().GetSnapshotPrefix(), ateompath.LocalCheckpointsDir(actorUID), checkpointDir, sandboxRec.SnapshotFiles); err != nil { + if err := s.copyLocalCheckpoint(gctx, req.GetLocalConfig().GetSnapshotPrefix(), s.paths.mapPath(ateompath.LocalCheckpointsDir(actorUID)), checkpointDir, sandboxRec.SnapshotFiles); err != nil { return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError) } } @@ -613,7 +629,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // Record the (manifest-pinned) sandbox binaries on-node so a subsequent // Checkpoint of this restored actor can re-pin the same version. - if err := writeSandboxRecord(actorUID, sandboxRec); err != nil { + if err := writeSandboxRecord(s.paths, actorUID, sandboxRec); err != nil { // Note: crash the actor right away, if we cannot write the sandbox record now, we will not be able to checkpoint it later. return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError) } @@ -698,7 +714,7 @@ func (s *AteomHerder) prepareOCIBundles( // Populate the per-actor identity directory that gets bind-mounted into // the application containers. Regenerated on every resume, so it carries // the correct per-actor name even when restoring from the golden snapshot. - identityDir := ateompath.ActorIdentityDirPath(actorUID) + identityDir := s.paths.mapPath(ateompath.ActorIdentityDirPath(actorUID)) if err := os.MkdirAll(identityDir, 0o755); err != nil { return fmt.Errorf("while creating actor identity dir: %w", err) } @@ -708,7 +724,7 @@ func (s *AteomHerder) prepareOCIBundles( // make directories for all durable-dir volumes for _, vol := range spec.GetVolumes() { if vol.GetType() == ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR { - volPath := ateompath.DurableDirVolumeMountPoint(actorUID, vol.GetName()) + volPath := s.paths.mapPath(ateompath.DurableDirVolumeMountPoint(actorUID, vol.GetName())) if err := os.MkdirAll(volPath, 0o700); err != nil { return fmt.Errorf("while creating %q: %w", volPath, err) } @@ -729,13 +745,14 @@ func (s *AteomHerder) prepareOCIBundles( if vol.GetType() == ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR { annotations["dev.gvisor.spec.mount.durabledir.type"] = "bind" annotations["dev.gvisor.spec.mount.durabledir.share"] = "container" - annotations["dev.gvisor.spec.mount.durabledir.source"] = ateompath.DurableDirVolumeMountPoint(actorUID, vol.GetName()) + annotations["dev.gvisor.spec.mount.durabledir.source"] = s.paths.mapPath(ateompath.DurableDirVolumeMountPoint(actorUID, vol.GetName())) } } if err := prepareOCIDirectory( gCtx, s.imageCache, + s.paths, actorUID, "pause", spec.GetPauseImage(), @@ -764,6 +781,7 @@ func (s *AteomHerder) prepareOCIBundles( if err := prepareOCIDirectory( gCtx, s.imageCache, + s.paths, actorUID, ctr.GetName(), ctr.GetImage(), @@ -792,11 +810,11 @@ func (s *AteomHerder) prepareOCIBundles( // dialAteom opens (or reuses) the gRPC connection to the target ateom // pod and returns an ateom client. func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ateompb.AteomClient, error) { - conn, err := s.ateomDialer.DialAteomPod(ctx, targetAteomUid) + client, err := s.ateomDialer.DialAteom(ctx, targetAteomUid) if err != nil { - return nil, fmt.Errorf("while getting ateom conn for %s: %w", targetAteomUid, err) + return nil, fmt.Errorf("while getting ateom client for %s: %w", targetAteomUid, err) } - return ateompb.NewAteomClient(conn), nil + return client, nil } // buildAteomWorkloadSpec projects the atelet-facing workload spec onto @@ -847,6 +865,14 @@ type AteomDialer struct { conns *lru.Cache } +func (d *AteomDialer) DialAteom(ctx context.Context, podUID string) (ateompb.AteomClient, error) { + conn, err := d.DialAteomPod(ctx, podUID) + if err != nil { + return nil, err + } + return ateompb.NewAteomClient(conn), nil +} + func (d *AteomDialer) DialAteomPod(ctx context.Context, podUID string) (*grpc.ClientConn, error) { key := podUID @@ -1043,7 +1069,7 @@ func writeFileAtomic(path string, data []byte, perm os.FileMode) error { return dir.Sync() } -func resetActorDirs(actorUID string) error { +func resetActorDirs(paths pathMapper, actorUID string) error { // Explicitly leave runsc logs dir untouched. // RemoveAllWritable, not os.RemoveAll: the bundle's upper dir can hold @@ -1052,7 +1078,7 @@ func resetActorDirs(actorUID string) error { // making them writable. (The rootfs itself is just an empty mountpoint // here: the overlay is mounted in the ateom pod's mount namespace, not // atelet's, and is detached by ateom at teardown.) - bundleDir := ateompath.OCIBundleDir(actorUID) + bundleDir := paths.mapPath(ateompath.OCIBundleDir(actorUID)) if err := imagecache.RemoveAllWritable(bundleDir); err != nil { return wrapFileSystemErr("while deleting bundle dir: %w", err) } @@ -1060,7 +1086,7 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating bundle dir: %w", err) } - runscDir := ateompath.RunSCStateDir(actorUID) + runscDir := paths.mapPath(ateompath.RunSCStateDir(actorUID)) if err := os.RemoveAll(runscDir); err != nil { return wrapFileSystemErr("while deleting runsc state dir: %w", err) } @@ -1068,7 +1094,7 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating runsc state dir: %w", err) } - pidFileDir := ateompath.PIDFileDir(actorUID) + pidFileDir := paths.mapPath(ateompath.PIDFileDir(actorUID)) if err := os.RemoveAll(pidFileDir); err != nil { return wrapFileSystemErr("while deleting PID file dir: %w", err) } @@ -1076,7 +1102,7 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating PID file dir: %w", err) } - checkpointDir := ateompath.CheckpointStateDir(actorUID) + checkpointDir := paths.mapPath(ateompath.CheckpointStateDir(actorUID)) if err := os.RemoveAll(checkpointDir); err != nil { return wrapFileSystemErr("while deleting checkpoint-state dir: %w", err) } @@ -1084,7 +1110,7 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating checkpoint-state dir: %w", err) } - restoreStateDir := ateompath.RestoreStateDir(actorUID) + restoreStateDir := paths.mapPath(ateompath.RestoreStateDir(actorUID)) if err := os.RemoveAll(restoreStateDir); err != nil { return wrapFileSystemErr("while deleting restore-state dir: %w", err) } @@ -1094,7 +1120,7 @@ func resetActorDirs(actorUID string) error { // World-readable (0o755): bind-mounted into the actor, whose workload // reads it through the gofer. - identityDir := ateompath.ActorIdentityDirPath(actorUID) + identityDir := paths.mapPath(ateompath.ActorIdentityDirPath(actorUID)) if err := os.RemoveAll(identityDir); err != nil { return wrapFileSystemErr("while deleting actor identity dir: %w", err) } @@ -1102,7 +1128,7 @@ func resetActorDirs(actorUID string) error { return wrapFileSystemErr("while creating actor identity dir: %w", err) } - durableDirVolumesMountDir := ateompath.DurableDirVolumeMountsDir(actorUID) + durableDirVolumesMountDir := paths.mapPath(ateompath.DurableDirVolumeMountsDir(actorUID)) if err := os.RemoveAll(durableDirVolumesMountDir); err != nil { return wrapFileSystemErr("while deleting durable-dir volumes mount dir: %w", err) } @@ -1112,7 +1138,7 @@ func resetActorDirs(actorUID string) error { // Do not call RemoveAll on volume directories in case the unmount failed. // We do not want to delete mount content. - volumesDir := ateompath.VolumesDir(actorUID) + volumesDir := paths.mapPath(ateompath.VolumesDir(actorUID)) entries, err := os.ReadDir(volumesDir) if err != nil && !os.IsNotExist(err) { return wrapFileSystemErr("while reading volumes dir: %w", err) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index ad0785f7b..895f3bc20 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -23,20 +23,44 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "syscall" "testing" "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" ) +func newAteomPathMapperForTest(t *testing.T) pathMapper { + t.Helper() + root := t.TempDir() + prefix := ateompath.BasePath + string(os.PathSeparator) + paths := pathMapper(func(path string) string { + if path == ateompath.BasePath { + return root + } + relative, ok := strings.CutPrefix(path, prefix) + if !ok { + return path + } + return filepath.Join(root, relative) + }) + if err := os.MkdirAll(paths.mapPath(ateompath.StaticFilesDir()), 0o700); err != nil { + t.Fatalf("creating static files dir: %v", err) + } + return paths +} + func TestWriteFileAtomic(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "actor-id") @@ -260,21 +284,19 @@ func TestValidateRestoreRequest(t *testing.T) { // prove the ordering, it plants a real file at the exact path an invalid hash // resolves to: a correctly-ordered fetchAsset validates first and returns an // error, while a regression that stats first would find this file and return it -// with a nil error, failing the test. StaticFilesDir is redirected to a temp -// dir so the planted path is writable and isolated. +// with a nil error, failing the test. The service uses a temp-rooted path +// layout so the derived static-files path is writable and isolated. func TestFetchAssetRejectsBadHash(t *testing.T) { - orig := ateompath.StaticFilesDir - ateompath.StaticFilesDir = t.TempDir() - t.Cleanup(func() { ateompath.StaticFilesDir = orig }) + paths := newAteomPathMapperForTest(t) // Invalid (8 chars, not 64) but separator-free, so it resolves to a normal - // filename inside the temp StaticFilesDir. + // filename inside the temp static-files directory. const badHash = "deadbeef" - if err := os.WriteFile(ateompath.RunSCBinaryPath(badHash), []byte("planted"), 0o755); err != nil { + if err := os.WriteFile(paths.mapPath(ateompath.RunSCBinaryPath(badHash)), []byte("planted"), 0o755); err != nil { t.Fatalf("planting cache file: %v", err) } - s := &AteomHerder{} + s := &AteomHerder{paths: paths} _, err := s.fetchAsset(context.Background(), assetEntry{SHA256: badHash}) if err == nil { t.Fatal("fetchAsset returned a cache hit for an invalid hash; validation must run before the os.Stat early return") @@ -304,16 +326,16 @@ func (fakeObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) // TestFetchAssetStreaming covers the streamed download: good asset cached, // over-cap rejected, hash mismatch rejected (failures leave no cache file). func TestFetchAssetStreaming(t *testing.T) { - origDir, origCap := ateompath.StaticFilesDir, maxAssetBytes - t.Cleanup(func() { ateompath.StaticFilesDir, maxAssetBytes = origDir, origCap }) + origCap := maxAssetBytes + t.Cleanup(func() { maxAssetBytes = origCap }) content := []byte("micro-vm kernel bytes") goodHash := fmt.Sprintf("%x", sha256.Sum256(content)) const url = "gs://test-bucket/asset" t.Run("good asset is cached", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() - s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} + paths := newAteomPathMapperForTest(t) + s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}, paths: paths} path, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) if err != nil { t.Fatalf("fetchAsset: %v", err) @@ -328,9 +350,9 @@ func TestFetchAssetStreaming(t *testing.T) { }) t.Run("over-cap asset rejected, cache not written", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() + paths := newAteomPathMapperForTest(t) maxAssetBytes = 4 // content is longer than this - s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} + s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}, paths: paths} _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) if err == nil { t.Fatal("fetchAsset accepted an over-cap asset") @@ -338,16 +360,16 @@ func TestFetchAssetStreaming(t *testing.T) { if !errors.Is(err, ateerrors.ReasonInvalidSandboxAsset) { t.Errorf("over-cap error not tagged terminal: %v", err) } - if _, err := os.Stat(ateompath.RunSCBinaryPath(goodHash)); !errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(paths.mapPath(ateompath.RunSCBinaryPath(goodHash))); !errors.Is(err, os.ErrNotExist) { t.Errorf("over-cap download left a file at the cache path (stat err = %v)", err) } }) t.Run("hash mismatch rejected, cache not written", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() + paths := newAteomPathMapperForTest(t) maxAssetBytes = origCap wrongHash := strings.Repeat("a", 64) // valid 64-hex format, wrong value - s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} + s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}, paths: paths} _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: wrongHash}) if err == nil { t.Fatal("fetchAsset accepted a hash mismatch") @@ -355,17 +377,17 @@ func TestFetchAssetStreaming(t *testing.T) { if !errors.Is(err, ateerrors.ReasonInvalidSandboxAsset) { t.Errorf("hash-mismatch error not tagged terminal: %v", err) } - if _, err := os.Stat(ateompath.RunSCBinaryPath(wrongHash)); !errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(paths.mapPath(ateompath.RunSCBinaryPath(wrongHash))); !errors.Is(err, os.ErrNotExist) { t.Errorf("mismatched download left a file at the cache path (stat err = %v)", err) } }) t.Run("missing object is terminal", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() + paths := newAteomPathMapperForTest(t) maxAssetBytes = origCap // The ategcs clients tag a missing object with ReasonFailedGetExternalObject. notFound := fmt.Errorf("%w: no such object", ateerrors.ReasonFailedGetExternalObject) - s := &AteomHerder{anonGCSClient: fakeObjectStorage{err: notFound}} + s := &AteomHerder{anonGCSClient: fakeObjectStorage{err: notFound}, paths: paths} _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) if !errors.Is(err, ateerrors.ReasonFailedGetExternalObject) { t.Errorf("missing-object error not tagged terminal: %v", err) @@ -381,9 +403,9 @@ func TestFetchAssetStreaming(t *testing.T) { }) t.Run("malformed url is terminal", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() + paths := newAteomPathMapperForTest(t) maxAssetBytes = origCap - s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} + s := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}, paths: paths} // Invalid percent-escape: url.Parse rejects it inside ategcs.Open, which // tags the failure with ReasonInvalidObjectURL. _, err := s.fetchAsset(context.Background(), assetEntry{URL: "gs://bucket/%zz", SHA256: goodHash}) @@ -393,9 +415,9 @@ func TestFetchAssetStreaming(t *testing.T) { }) t.Run("network error stays untagged (retriable)", func(t *testing.T) { - ateompath.StaticFilesDir = t.TempDir() + paths := newAteomPathMapperForTest(t) maxAssetBytes = origCap - s := &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("connection refused")}} + s := &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("connection refused")}, paths: paths} _, err := s.fetchAsset(context.Background(), assetEntry{URL: url, SHA256: goodHash}) if err == nil { t.Fatal("fetchAsset accepted a failing open") @@ -458,6 +480,150 @@ func TestRPCBoundariesReject(t *testing.T) { }) } +type fakeImageEnsurer struct{} + +func (fakeImageEnsurer) EnsureImage(context.Context, string) (*imagecache.Image, error) { + return &imagecache.Image{Config: v1.Config{}}, nil +} + +type fakeAteomClient struct { + runReq *ateompb.RunWorkloadRequest + checkpointReq *ateompb.CheckpointWorkloadRequest + restoreReq *ateompb.RestoreWorkloadRequest + paths pathMapper +} + +func (c *fakeAteomClient) RunWorkload(_ context.Context, req *ateompb.RunWorkloadRequest, _ ...grpc.CallOption) (*ateompb.RunWorkloadResponse, error) { + c.runReq = req + return &ateompb.RunWorkloadResponse{}, nil +} + +func (c *fakeAteomClient) CheckpointWorkload(_ context.Context, req *ateompb.CheckpointWorkloadRequest, _ ...grpc.CallOption) (*ateompb.CheckpointWorkloadResponse, error) { + c.checkpointReq = req + const snapshotFile = "checkpoint.img" + if err := os.WriteFile(filepath.Join(c.paths.mapPath(ateompath.CheckpointStateDir(req.GetActorUid())), snapshotFile), []byte("snapshot"), 0o600); err != nil { + return nil, fmt.Errorf("writing fake checkpoint: %w", err) + } + return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: []string{snapshotFile}}, nil +} + +func (c *fakeAteomClient) RestoreWorkload(_ context.Context, req *ateompb.RestoreWorkloadRequest, _ ...grpc.CallOption) (*ateompb.RestoreWorkloadResponse, error) { + c.restoreReq = req + return &ateompb.RestoreWorkloadResponse{}, nil +} + +type fakeAteomDialer struct { + client ateompb.AteomClient + targets []string +} + +func (d *fakeAteomDialer) DialAteom(_ context.Context, target string) (ateompb.AteomClient, error) { + d.targets = append(d.targets, target) + return d.client, nil +} + +// TestRPCBoundariesAccept runs a successful local lifecycle through the public +// atelet RPC methods. The fake dependencies stop at the image-fetch and ateom +// client boundaries; filesystem setup, request projection, checkpoint movement, +// and sandbox record handling all use the production paths. +func TestRPCBoundariesAccept(t *testing.T) { + paths := newAteomPathMapperForTest(t) + runscSum := fmt.Sprintf("%x", sha256.Sum256([]byte("fake runsc"))) + runscPath := paths.mapPath(ateompath.RunSCBinaryPath(runscSum)) + if err := os.WriteFile(runscPath, []byte("fake runsc"), 0o755); err != nil { + t.Fatalf("seeding runsc cache: %v", err) + } + + client := &fakeAteomClient{paths: paths} + dialer := &fakeAteomDialer{client: client} + // GCS clients stay nil: the runsc asset is pre-seeded above, so + // ensureSandboxAssets never downloads, and checkpoint/restore use local + // snapshots, so no path touches object storage. + s := &AteomHerder{ateomDialer: dialer, imageCache: fakeImageEnsurer{}, paths: paths} + ctx := context.Background() + const snapshotPrefix = "snapshot-1" + + runReq := validRunRequest() + runReq.Spec.PauseImage = "pause:test" + runReq.Spec.Containers[0].Image = "worker:test" + runReq.Spec.Containers[0].Command = []string{"/worker"} + runReq.SandboxAssets = &ateletpb.SandboxAssets{ + SandboxClass: "gvisor", + Assets: map[string]*ateletpb.ArchAssets{ + runtime.GOARCH: { + Files: map[string]*ateletpb.AssetFile{ + "runsc": {Url: "gs://unused/runsc", Sha256: runscSum}, + }, + }, + }, + } + + t.Run("Run", func(t *testing.T) { + if _, err := s.Run(ctx, runReq); err != nil { + t.Fatalf("Run: %v", err) + } + if client.runReq == nil { + t.Fatal("Run did not call ateom") + } + if got := client.runReq.GetRunscPath(); got != runscPath { + t.Errorf("runsc path = %q, want %q", got, runscPath) + } + if got := client.runReq.GetActorUid(); got != runReq.GetActorUid() { + t.Errorf("actor UID = %q, want %q", got, runReq.GetActorUid()) + } + }) + + checkpointReq := validCheckpointRequest() + checkpointReq.Spec = runReq.Spec + checkpointReq.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL + checkpointReq.Config = &ateletpb.CheckpointRequest_LocalConfig{ + LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotPrefix: snapshotPrefix}, + } + t.Run("Checkpoint", func(t *testing.T) { + if _, err := s.Checkpoint(ctx, checkpointReq); err != nil { + t.Fatalf("Checkpoint: %v", err) + } + if client.checkpointReq == nil { + t.Fatal("Checkpoint did not call ateom") + } + if got := client.checkpointReq.GetRunscPath(); got != runscPath { + t.Errorf("runsc path = %q, want %q", got, runscPath) + } + localSnapshot := filepath.Join(paths.mapPath(ateompath.LocalCheckpointsDir(checkpointReq.GetActorUid())), snapshotPrefix, "checkpoint.img") + if got, err := os.ReadFile(localSnapshot); err != nil { + t.Errorf("reading local snapshot: %v", err) + } else if string(got) != "snapshot" { + t.Errorf("local snapshot = %q, want %q", got, "snapshot") + } + }) + + restoreReq := validRestoreRequest() + restoreReq.Spec = runReq.Spec + restoreReq.Type = ateletpb.CheckpointType_CHECKPOINT_TYPE_LOCAL + restoreReq.Config = &ateletpb.RestoreRequest_LocalConfig{ + LocalConfig: &ateletpb.LocalCheckpointConfiguration{SnapshotPrefix: snapshotPrefix}, + } + t.Run("Restore", func(t *testing.T) { + if _, err := s.Restore(ctx, restoreReq); err != nil { + t.Fatalf("Restore: %v", err) + } + if client.restoreReq == nil { + t.Fatal("Restore did not call ateom") + } + if got := client.restoreReq.GetRunscPath(); got != runscPath { + t.Errorf("runsc path = %q, want %q", got, runscPath) + } + if got := client.restoreReq.GetActorUid(); got != restoreReq.GetActorUid() { + t.Errorf("actor UID = %q, want %q", got, restoreReq.GetActorUid()) + } + }) + + wantTargets := []string{runReq.GetTargetAteomUid(), checkpointReq.GetTargetAteomUid(), restoreReq.GetTargetAteomUid()} + if diff := cmp.Diff(wantTargets, dialer.targets); diff != "" { + t.Errorf("dial targets mismatch (-want +got):\n%s", diff) + } +} + func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) { in := &ateletpb.WorkloadSpec{ PauseImage: "pause", diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index e9c496018..79c7488ce 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -50,14 +50,18 @@ const ( ActorIDFileName = "actor-id" ) -func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, command, args []string, env []string, annotations map[string]string, netns string, identityDir string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) error { +type imageEnsurer interface { + EnsureImage(context.Context, string) (*imagecache.Image, error) +} + +func prepareOCIDirectory(ctx context.Context, imageCache imageEnsurer, paths pathMapper, actorUID, containerName, ref string, command, args []string, env []string, annotations map[string]string, netns string, identityDir string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) error { tracer := otel.Tracer("prepareOCIDirectory") ctx, span := tracer.Start(ctx, "prepareOCIDirectory") span.SetAttributes(attribute.String("image", ref)) defer span.End() - bundlePath := ateompath.OCIBundlePath(actorUID, containerName) + bundlePath := paths.mapPath(ateompath.OCIBundlePath(actorUID, containerName)) // Clear any previous bundle contents (belt and suspenders: resetActorDirs // already wiped the bundle dir on the Run/Restore path). @@ -108,7 +112,7 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto return fmt.Errorf("while writing overlay spec: %w", err) } - ociSpec := buildActorOCISpec(actorUID, resolvedArgs, resolvedEnv, annotations, netns, identityDir, volumes, volumeMounts) + ociSpec := buildActorOCISpec(paths, actorUID, resolvedArgs, resolvedEnv, annotations, netns, identityDir, volumes, volumeMounts) ociSpecBytes, err := json.MarshalIndent(ociSpec, "", " ") if err != nil { return fmt.Errorf("while marshaling OCI spec: %w", err) @@ -185,7 +189,7 @@ func resolveProcessArgs(imageCfg *v1.Config, command, args []string) ([]string, // When identityDir is non-empty it adds a read-only bind mount of that host // directory at IdentityMountPath so the actor can read its own ID (see // IdentityMountPath for why this is a bind mount rather than env vars). -func buildActorOCISpec(actorUID string, args []string, env []string, annotations map[string]string, netns string, identityDir string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) *specs.Spec { +func buildActorOCISpec(paths pathMapper, actorUID string, args []string, env []string, annotations map[string]string, netns string, identityDir string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) *specs.Spec { mounts := []specs.Mount{ { Destination: "/proc", @@ -303,9 +307,9 @@ func buildActorOCISpec(actorUID string, args []string, env []string, annotations var srcPath string switch volumeTypes[vm.GetName()] { case ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR: - srcPath = ateompath.DurableDirVolumeMountPoint(actorUID, vm.GetName()) + srcPath = paths.mapPath(ateompath.DurableDirVolumeMountPoint(actorUID, vm.GetName())) case ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL: - srcPath = ateompath.VolumeHostPath(actorUID, vm.GetName()) + srcPath = paths.mapPath(ateompath.VolumeHostPath(actorUID, vm.GetName())) default: continue } diff --git a/cmd/atelet/oci_test.go b/cmd/atelet/oci_test.go index 433c082c3..754fe4688 100644 --- a/cmd/atelet/oci_test.go +++ b/cmd/atelet/oci_test.go @@ -28,6 +28,7 @@ import ( // With an identity dir, a read-only bind mount appears at IdentityMountPath. func TestBuildActorOCISpec_IdentityMount(t *testing.T) { spec := buildActorOCISpec( + nil, "actor_uid", []string{"/app"}, []string{"FOO=bar"}, @@ -194,7 +195,7 @@ func TestResolveProcessArgs(t *testing.T) { // Without an identity dir (the pause container), no identity mount appears. func TestBuildActorOCISpec_NoIdentityMountForPause(t *testing.T) { - bare := buildActorOCISpec("actor_uid", []string{"/pause"}, nil, nil, "/run/netns/x", "", nil, nil) + bare := buildActorOCISpec(nil, "actor_uid", []string{"/pause"}, nil, nil, "/run/netns/x", "", nil, nil) for _, m := range bare.Mounts { if m.Destination == IdentityMountPath { t.Errorf("identity mount must be absent when identityDir is empty") @@ -215,6 +216,7 @@ func TestBuildActorOCISpec_DurableDirVolumeMounts(t *testing.T) { {Name: "cache", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, } spec := buildActorOCISpec( + nil, actorUID, []string{"/app"}, nil, nil, "/run/netns/x", diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 0154837cb..48b4e41e3 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -93,7 +93,7 @@ func recordFromRequest(sa *ateletpb.SandboxAssets) (*sandboxAssetsRecord, error) // the micro-VM runtime has several (kata-shim, cloud-hypervisor, ...). Assets are // cached, so re-fetching at Checkpoint/Restore is a no-op once present. func (s *AteomHerder) ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) { - if err := os.MkdirAll(ateompath.StaticFilesDir, 0o700); err != nil { + if err := os.MkdirAll(s.paths.mapPath(ateompath.StaticFilesDir()), 0o700); err != nil { if isTerminalFileSystemErr(err) { return nil, fmt.Errorf("%w: while creating static files dir: %w", ateerrors.ReasonTerminalFileSystemError, err) } @@ -122,7 +122,7 @@ func (s *AteomHerder) fetchAsset(ctx context.Context, entry assetEntry) (string, return "", wrapFileSystemErr("while validating asset hash", err) } - localPath := ateompath.RunSCBinaryPath(entry.SHA256) + localPath := s.paths.mapPath(ateompath.RunSCBinaryPath(entry.SHA256)) _, err := os.Stat(localPath) if err == nil { return localPath, nil @@ -206,12 +206,12 @@ func (s *AteomHerder) openAsset(ctx context.Context, url string) (io.ReadCloser, // writeSandboxRecord persists the actor's running sandbox assets on-node so a // later Checkpoint (whose request no longer carries the sandbox config) can // re-fetch the same binaries and pin them into the snapshot manifest. -func writeSandboxRecord(actorUID string, rec *sandboxAssetsRecord) error { +func writeSandboxRecord(paths pathMapper, actorUID string, rec *sandboxAssetsRecord) error { data, err := json.Marshal(rec) if err != nil { return wrapFileSystemErr("while marshaling sandbox record", err) } - path := ateompath.ActorSandboxAssetsFile(actorUID) + path := paths.mapPath(ateompath.ActorSandboxAssetsFile(actorUID)) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return wrapFileSystemErr("while creating actor dir", err) } @@ -223,8 +223,8 @@ func writeSandboxRecord(actorUID string, rec *sandboxAssetsRecord) error { // readSandboxRecord loads the actor's on-node sandbox record written at // Run/Restore. -func readSandboxRecord(actorUID string) (*sandboxAssetsRecord, error) { - path := ateompath.ActorSandboxAssetsFile(actorUID) +func readSandboxRecord(paths pathMapper, actorUID string) (*sandboxAssetsRecord, error) { + path := paths.mapPath(ateompath.ActorSandboxAssetsFile(actorUID)) data, err := os.ReadFile(path) if err != nil { return nil, wrapFileSystemErr("while reading sandbox record", err) diff --git a/cmd/atelet/volumes.go b/cmd/atelet/volumes.go index 3b0ae3453..85f93a555 100644 --- a/cmd/atelet/volumes.go +++ b/cmd/atelet/volumes.go @@ -43,7 +43,7 @@ func (s *AteomHerder) mountExternalVolumes(ctx context.Context, actorUID string, if ext == nil { continue } - hostPath := ateompath.VolumeHostPath(actorUID, vol.GetName()) + hostPath := s.paths.mapPath(ateompath.VolumeHostPath(actorUID, vol.GetName())) if err := os.MkdirAll(hostPath, 0o750); err != nil { return fmt.Errorf("failed to create mount point %q: %w", hostPath, err) } @@ -64,7 +64,7 @@ func (s *AteomHerder) unmountExternalVolumes(ctx context.Context, actorUID strin if ext == nil { continue } - hostPath := ateompath.VolumeHostPath(actorUID, vol.GetName()) + hostPath := s.paths.mapPath(ateompath.VolumeHostPath(actorUID, vol.GetName())) slog.InfoContext(ctx, "Unmounting volume", slog.String("volume_id", ext.GetStorageVolumeId()), slog.String("host_path", hostPath)) if err := getVolumePlugin().UnmountVolume(ctx, ext.GetStorageVolumeId(), hostPath); err != nil { slog.ErrorContext(ctx, "failed to unmount volume", slog.String("volume_id", ext.GetStorageVolumeId()), slog.String("host_path", hostPath), slog.Any("error", err)) diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index a6f9deb8f..aeb644942 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -19,25 +19,25 @@ import ( "path/filepath" ) -const ( - // The base path. This is both the path of the root shared folder on the - // host filesystem, and when it is mounted into ateom and atelet containers. - BasePath = "/var/lib/ateom-gvisor" -) +// BasePath is both the path of the root shared folder on the host filesystem, +// and where it is mounted into ateom and atelet containers. +const BasePath = "/var/lib/ateom-gvisor" -var ( - // StaticFilesDir holds things like downloaded runsc binaries. - StaticFilesDir = filepath.Join(BasePath, "static-files") +// StaticFilesDir holds things like downloaded runsc binaries. +func StaticFilesDir() string { + return filepath.Join(BasePath, "static-files") +} - // ImageCacheDir is the node-local OCI image layer cache (see - // internal/imagecache). It lives under BasePath so the cached layer - // directories are visible at the same path in atelet (which writes them) - // and in every ateom pod (which mounts them as overlay lowerdirs). - ImageCacheDir = filepath.Join(BasePath, "image-cache") -) +// ImageCacheDir is the node-local OCI image layer cache (see +// internal/imagecache). It lives under BasePath so the cached layer +// directories are visible at the same path in atelet (which writes them) +// and in every ateom pod (which mounts them as overlay lowerdirs). +func ImageCacheDir() string { + return filepath.Join(BasePath, "image-cache") +} func RunSCBinaryPath(sha256 string) string { - return filepath.Join(StaticFilesDir, "runsc-"+sha256) + return filepath.Join(StaticFilesDir(), "runsc-"+sha256) } func AteomPath(podUID string) string {