diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 09a41f410..1c49a8632 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -486,7 +486,7 @@ func toAteomSnapshotScope(scope ateletpb.SnapshotScope) ateompb.SnapshotScope { func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { localCheckpointPath := filepath.Join(ateompath.LocalCheckpointsDir(req.GetActorUid()), req.GetLocalConfig().GetSnapshotPrefix()) - if err := os.MkdirAll(localCheckpointPath, 0o700); err != nil { + if err := ensureParentDirs(localCheckpointPath, rec.SnapshotFiles); err != nil { return fmt.Errorf("while creating local checkpoint directory: %w", err) } @@ -620,6 +620,15 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) if goldenRec.SandboxClass != sandboxRec.SandboxClass { return nil, status.Errorf(codes.FailedPrecondition, "golden snapshot sandbox class %q does not match actor snapshot sandbox class %q", goldenRec.SandboxClass, sandboxRec.SandboxClass) } + // gVisor checkpoint images are runsc-version-coupled: the actor's + // data fs image and the golden's memory image can only be combined + // when both were produced by the same pinned runsc. The micro-VM + // data half is a plain tar, so no such coupling there. + if sandboxRec.SandboxClass == sandboxClassGvisor { + if got, want := gvisorRuntimeAssetSHA(sandboxRec), gvisorRuntimeAssetSHA(goldenRec); got != want { + return nil, status.Errorf(codes.FailedPrecondition, "actor snapshot gVisor runtime (sha256 %s) does not match golden snapshot gVisor runtime (sha256 %s); the snapshots cannot be combined", got, want) + } + } } // The record whose pinned binaries run the restored workload: the golden's @@ -633,6 +642,24 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) runtimeRec = goldenRec } + // Where a DATA_ON_GOLDEN restore stages each half, so the restore dir + // ends up looking like a Full snapshot whose durable data is the ACTOR's. + // Micro-VM: one flat folder — the actor's files shadow the golden's + // same-named files (its durable tar replaces the golden's). + // gVisor: the golden's split checkpoint keeps memory files at the top level + // and the durable fs image under fs/; the golden's fs/ contents are skipped + // and the actor's data snapshot (a flat fscheckpoint image set) is staged + // into fs/ instead, overriding it wholesale. + actorDstDir := checkpointDir + var goldenFiles []string + if goldenRec != nil { + goldenFiles = goldenOnlyFiles(sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles) + if sandboxRec.SandboxClass == sandboxClassGvisor { + actorDstDir = filepath.Join(checkpointDir, gvisorSplitFsSubdir) + goldenFiles = filesOutsideDir(goldenRec.SnapshotFiles, gvisorSplitFsSubdir) + } + } + // Download the memory snapshot and prepare the sandbox assets + OCI bundle // CONCURRENTLY. They are independent — only the final ateom.RestoreWorkload // needs both — so overlapping the GCS download (~0.5s warm) with the asset @@ -650,7 +677,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) if goldenRec == nil { return fmt.Errorf("no golden snapshot record for a %s restore", req.GetScope()) } - if err := s.downloadCombinedCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), req.GetGoldenSnapshotUriPrefix(), checkpointDir, sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles); err != nil { + if err := s.downloadCombinedCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), actorDstDir, sandboxRec.SnapshotFiles, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenFiles); err != nil { return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError) } } else if err := s.downloadExternalCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), checkpointDir, sandboxRec.SnapshotFiles); err != nil { @@ -666,14 +693,14 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // the golden's from object storage, concurrently. gLocal, gLocalCtx := errgroup.WithContext(gctx) gLocal.Go(func() error { - if err := s.copyLocalCheckpoint(gLocalCtx, req.GetLocalConfig().GetSnapshotPrefix(), ateompath.LocalCheckpointsDir(actorUID), checkpointDir, sandboxRec.SnapshotFiles); err != nil { + if err := s.copyLocalCheckpoint(gLocalCtx, req.GetLocalConfig().GetSnapshotPrefix(), ateompath.LocalCheckpointsDir(actorUID), actorDstDir, sandboxRec.SnapshotFiles); err != nil { return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError) } return nil }) if combineWithGolden { gLocal.Go(func() error { - if err := s.downloadExternalCheckpoint(gLocalCtx, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenOnlyFiles(sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles)); err != nil { + if err := s.downloadExternalCheckpoint(gLocalCtx, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenFiles); err != nil { return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError) } return nil @@ -749,6 +776,9 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) } func (s *AteomHerder) copyLocalCheckpoint(ctx context.Context, snapshotPrefix string, srcDir, dstDir string, files []string) error { + if err := ensureParentDirs(dstDir, files); err != nil { + return err + } for _, fileName := range files { if ctx.Err() != nil { return fmt.Errorf("context cancelled: %w", ctx.Err()) @@ -807,28 +837,65 @@ func goldenOnlyFiles(actorFiles, goldenFiles []string) []string { return rest } -// downloadCombinedCheckpoint stages a DATA_ON_GOLDEN restore set into dstDir -// as a single folder: every file of the actor's own snapshot (the durable-dir -// data) plus the golden snapshot's files the actor's set does not shadow, so -// the result looks like a Full snapshot whose durable-dir data is the actor's. -func (s *AteomHerder) downloadCombinedCheckpoint(ctx context.Context, actorPrefix, goldenPrefix, dstDir string, actorFiles, goldenFiles []string) error { +// downloadCombinedCheckpoint stages both halves of a DATA_ON_GOLDEN restore +// concurrently: the actor's own snapshot files (the durable-dir data) into +// actorDstDir and the given golden snapshot files into goldenDstDir. The +// caller decides the layout: one flat folder with the actor's files +// shadowing the golden's (micro-VM), or the actor's files re-rooted into +// the golden checkpoint's fs/ subfolder (gVisor). +func (s *AteomHerder) downloadCombinedCheckpoint(ctx context.Context, actorPrefix, actorDstDir string, actorFiles []string, goldenPrefix, goldenDstDir string, goldenFiles []string) error { g, gctx := errgroup.WithContext(ctx) g.Go(func() error { - return s.downloadExternalCheckpoint(gctx, actorPrefix, dstDir, actorFiles) + return s.downloadExternalCheckpoint(gctx, actorPrefix, actorDstDir, actorFiles) }) g.Go(func() error { - return s.downloadExternalCheckpoint(gctx, goldenPrefix, dstDir, goldenOnlyFiles(actorFiles, goldenFiles)) + return s.downloadExternalCheckpoint(gctx, goldenPrefix, goldenDstDir, goldenFiles) }) return g.Wait() } +// filesOutsideDir returns the names not under the given top-level subdir. A +// gVisor DATA_ON_GOLDEN restore fetches the golden's memory files (top +// level) but not its fs/ contents, which the actor's data replaces. +func filesOutsideDir(files []string, subdir string) []string { + kept := make([]string, 0, len(files)) + for _, f := range files { + if !strings.HasPrefix(f, subdir+"/") { + kept = append(kept, f) + } + } + return kept +} + +// ensureParentDirs creates, once each, the distinct parent directories the +// given (possibly subdir-relative) +// snapshot file names need under dstDir. +func ensureParentDirs(dstDir string, files []string) error { + made := make(map[string]struct{}, 2) + for _, f := range files { + dir := filepath.Dir(filepath.Join(dstDir, f)) + if _, ok := made[dir]; ok { + continue + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("while creating snapshot subdirectory %q: %w", dir, err) + } + made[dir] = struct{}{} + } + return nil +} + func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUriPrefix string, dstDir string, files []string) error { prefix := strings.TrimSuffix(snapshotUriPrefix, "/") + if err := ensureParentDirs(dstDir, files); err != nil { + return err + } g, gCtx := errgroup.WithContext(ctx) for _, fileName := range files { fileName := fileName local := filepath.Join(dstDir, fileName) g.Go(func() error { + slog.InfoContext(ctx, "@@@@@ [downloadExternalCheckpoint #1]", slog.String("file", prefix+"/"+fileName+".zstd"), slog.String("local", local)) if err := ategcs.FetchLocalFileFromGCSWithZstd(gCtx, s.gcsClient, prefix+"/"+fileName+".zstd", local); err != nil { return fmt.Errorf("while downloading %s from GCS: %w", fileName, err) } diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 88058cffe..8d0408ec5 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -786,10 +786,11 @@ func (m mapObjectStorage) GetObject(_ context.Context, bucket, object string) (i func (mapObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } -// TestDownloadCombinedCheckpoint verifies a DataOnGolden restore stages one -// folder holding the actor snapshot's durable-dir tar and the golden -// snapshot's remaining files — and that the golden's own durable-dir tar is -// the one that loses the name collision. +// TestDownloadCombinedCheckpoint verifies the micro-VM DataOnGolden layout: +// one flat folder holding the actor snapshot's durable-dir tar and the golden +// snapshot's remaining files — the caller shadows the golden's own +// durable-dir tar out of the golden file list (goldenOnlyFiles), so the +// actor's tar wins the name collision. func TestDownloadCombinedCheckpoint(t *testing.T) { zstdBytes := func(t *testing.T, s string) []byte { t.Helper() @@ -816,12 +817,11 @@ func TestDownloadCombinedCheckpoint(t *testing.T) { s := &AteomHerder{gcsClient: store} dstDir := t.TempDir() + actorFiles := []string{"durable-dir.tar"} + goldenFiles := []string{"config.json", "memory-ranges", "durable-dir.tar"} err := s.downloadCombinedCheckpoint(context.Background(), - "gs://bucket/actors/1/snapshots/2/", - "gs://bucket/ate-golden/snapshots/1/", - dstDir, - []string{"durable-dir.tar"}, - []string{"config.json", "memory-ranges", "durable-dir.tar"}) + "gs://bucket/actors/1/snapshots/2/", dstDir, actorFiles, + "gs://bucket/ate-golden/snapshots/1/", dstDir, goldenOnlyFiles(actorFiles, goldenFiles)) if err != nil { t.Fatalf("downloadCombinedCheckpoint: %v", err) } @@ -988,3 +988,69 @@ func TestDrainOnShutdownForceStopsAfterTimeout(t *testing.T) { t.Fatal("readiness should be not-ready after drain") } } + +// TestDownloadCombinedCheckpointGvisorFsOverride verifies the gVisor +// DataOnGolden layout: the golden split checkpoint's memory files land at +// the restore dir's top level (its own fs/ contents are excluded by +// filesOutsideDir), while the actor's flat fscheckpoint image set is +// re-rooted into the fs/ subfolder, overriding it wholesale — and the +// subfolder is created on demand. +func TestDownloadCombinedCheckpointGvisorFsOverride(t *testing.T) { + zstdBytes := func(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf) + if err != nil { + t.Fatalf("zstd.NewWriter: %v", err) + } + if _, err := zw.Write([]byte(s)); err != nil { + t.Fatalf("zstd write: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("zstd close: %v", err) + } + return buf.Bytes() + } + + store := mapObjectStorage{objects: map[string][]byte{ + "bucket/actors/1/snapshots/2/multitar.img.zstd": zstdBytes(t, "actor data fs image"), + "bucket/ate-golden/snapshots/1/checkpoint.img.zstd": zstdBytes(t, "golden memory"), + "bucket/ate-golden/snapshots/1/pages.img.zstd": zstdBytes(t, "golden pages"), + "bucket/ate-golden/snapshots/1/fs/multitar.img.zstd": zstdBytes(t, "golden fs image (must not be downloaded)"), + }} + s := &AteomHerder{gcsClient: store} + + dstDir := t.TempDir() + actorFiles := []string{"multitar.img"} + goldenFiles := []string{"checkpoint.img", "fs/multitar.img", "pages.img"} + err := s.downloadCombinedCheckpoint(context.Background(), + "gs://bucket/actors/1/snapshots/2/", filepath.Join(dstDir, "fs"), actorFiles, + "gs://bucket/ate-golden/snapshots/1/", dstDir, filesOutsideDir(goldenFiles, "fs")) + if err != nil { + t.Fatalf("downloadCombinedCheckpoint: %v", err) + } + + for path, content := range map[string]string{ + filepath.Join(dstDir, "checkpoint.img"): "golden memory", + filepath.Join(dstDir, "pages.img"): "golden pages", + filepath.Join(dstDir, "fs", "multitar.img"): "actor data fs image", + } { + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s): %v", path, err) + } + if string(got) != content { + t.Errorf("%s content = %q, want %q", path, got, content) + } + } +} + +// TestFilesOutsideDir verifies the golden-file filter for the gVisor +// fs-override layout, including that a prefix-similar name (fsx) survives. +func TestFilesOutsideDir(t *testing.T) { + got := filesOutsideDir([]string{"checkpoint.img", "fs/multitar.img", "fs/pages.img", "fsx.img"}, "fs") + want := []string{"checkpoint.img", "fsx.img"} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("filesOutsideDir diff (-want +got):\n%s", diff) + } +} diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index 11dac9e4a..a2aba32df 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -50,6 +50,14 @@ const sandboxManifestName = "manifest.json" var maxAssetBytes int64 = 8 << 30 const ( + // sandboxClassGvisor mirrors the gVisor SandboxClass value recorded in a + // manifest's sandboxClass field (see atev1alpha1.SandboxClassGvisor; + // atelet does not depend on the CRD API package). Not to be confused + // with gvisorAssetName below: the class names the runtime family, the + // asset name keys one entry of the class's asset map, and the two only + // coincidentally share the value "gvisor". + sandboxClassGvisor = "gvisor" + // gvisorAssetName is the gVisor release tarball asset (gvisor.tar.bz2). The // tarball carries `runsc` together with the `gvisor-bin/` helper binaries, // so it is extracted into a content-addressed directory rather than as a @@ -58,6 +66,12 @@ const ( // runscAssetName is the legacy single-binary gVisor asset. runscAssetName = "runsc" + + // gvisorSplitFsSubdir is the subfolder of a gVisor split checkpoint + // (`runsc checkpoint -split-fscheckpoint`) holding the durable-dir fs + // image; the memory snapshot files sit at the checkpoint's top level, and + // `runsc restore -split-fsrestore` reads the fs half from this subfolder. + gvisorSplitFsSubdir = "fs" ) // assetEntry is one content-addressed sandbox asset (url + sha256). @@ -142,6 +156,17 @@ func (s *AteomHerder) ensureSandboxAssets(ctx context.Context, rec *sandboxAsset return paths, nil } +// gvisorRuntimeAssetSHA returns the sha256 pinning a record's gVisor runtime: +// the release tarball asset, or the legacy bare runsc binary for records +// written before the tarball mechanism. Used to require that the two halves +// of a DATA_ON_GOLDEN combine were produced by the same runsc. +func gvisorRuntimeAssetSHA(rec *sandboxAssetsRecord) string { + if a, ok := rec.Assets[gvisorAssetName]; ok { + return a.SHA256 + } + return rec.Assets[runscAssetName].SHA256 +} + // runscPathFor returns the local path of the gVisor `runsc` binary from a // fetched asset-path map, or "" if the runtime has none (e.g. micro-VM). func runscPathFor(paths map[string]string) string { diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 752acf2d3..c23d88ca9 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -20,10 +20,12 @@ import ( "context" "errors" "fmt" + "io/fs" "log/slog" "net" "net/url" "os" + "path/filepath" "sort" "strings" "sync" @@ -439,15 +441,22 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // listSnapshotFiles returns the (relative) names of regular files directly under // dir, which atelet ships to object storage as the snapshot. func listSnapshotFiles(dir string) ([]string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err - } var files []string - for _, e := range entries { - if e.Type().IsRegular() { - files = append(files, e.Name()) + if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err } + if !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + files = append(files, rel) + return nil + }); err != nil { + return nil, err } sort.Strings(files) return files, nil @@ -529,18 +538,30 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while composing pause rootfs: %w", err) } + fsRestoreArgs := func(dir string) []string { + if _, err := os.Stat(filepath.Join(dir, "fs")); err == nil { + return []string{"--fs-restore-image-path", dir + "/fs"} + } + return nil + } + switch req.GetScope() { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: // Create and restore pause container - if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", []string{"--fs-restore-image-path", checkpointDir}); err != nil { + if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", fsRestoreArgs(checkpointDir)); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: - // Create and restore pause container - if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil { + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN: + // DATA_ON_GOLDEN restores exactly like FULL: atelet staged the golden + // checkpoint's memory files at the restore dir's top level and the + // ACTOR's data fs image in its fs/ subfolder, which is where + // `runsc restore -split-fsrestore` reads the fs half from anyway. + // Create and restore pause container. + if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", fsRestoreArgs(checkpointDir)); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } if err := rcmd.cmdRestore(ctx, os.Stdout, "pause", checkpointDir); err != nil { @@ -569,7 +590,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err := rcmd.cmdStart(ctx, pw, ac.GetName()); err != nil { return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err) } - case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: + case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN: if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) } diff --git a/cmd/ateom-gvisor/runsc.go b/cmd/ateom-gvisor/runsc.go index 003bd861a..5ca82609b 100644 --- a/cmd/ateom-gvisor/runsc.go +++ b/cmd/ateom-gvisor/runsc.go @@ -164,6 +164,7 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath "-root", ateompath.RunSCStateDir(r.actorUID), "checkpoint", "-image-path", checkpointPath, + "-split-fscheckpoint", containerName, // Name of the container ) cmd.Stdout = os.Stdout @@ -240,6 +241,8 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch "restore", "-bundle", ateompath.OCIBundlePath(r.actorUID, containerName), "-image-path", checkpointPath, + "-fs-restore-image-path", filepath.Join(checkpointPath, "fs"), + "-split-fsrestore", "-pid-file", ateompath.PIDFilePath(r.actorUID, containerName), "-background", "-direct", diff --git a/demos/counter/counter.go b/demos/counter/counter.go index 107bef68d..1ec37f229 100644 --- a/demos/counter/counter.go +++ b/demos/counter/counter.go @@ -18,11 +18,7 @@ package main import ( "context" - "crypto/rand" - "crypto/sha256" - "encoding/base64" "fmt" - "io" "log/slog" "net" "net/http" @@ -31,7 +27,6 @@ import ( "strconv" "sync" "sync/atomic" - "time" "github.com/spf13/pflag" ) @@ -115,59 +110,20 @@ func main() { w.Write([]byte("ok\n")) }) - go func() { - slog.InfoContext(ctx, "Starting counter server on port 80") - if err := http.ListenAndServe(":80", defaultMux); err != nil { - slog.ErrorContext(ctx, "Error starting server", slog.Any("err", err)) - os.Exit(1) - } - }() - - // Write some random data to a file in the root filesystem, to test - // filesystem checkpoint/restore. - if err := writeRandomFile(); err != nil { - slog.InfoContext(ctx, "Error writing random file", slog.Any("err", err)) - } else { - slog.InfoContext(ctx, "Wrote content to random file", slog.String("fshash", hashRandomFile())) - } + // TODO(dberkov): remove the temporary workaround for adding a file to durDir + // othereise gVisor does not take golden snapshot correclty. + incrementFileCounter(filepath.Join(*fileCounterDirectory, "tmp.txt")) ready.Store(true) slog.InfoContext(ctx, "Readyz now reports OK") - count := 0 - slog.InfoContext(ctx, "Count", slog.Int("count", count), slog.String("fshash", hashRandomFile())) - count++ - - for range time.Tick(10 * time.Second) { - // TODO: Test outbound connectivity by pinging google.com - slog.InfoContext(ctx, "Count", slog.Int("count", count), slog.String("fshash", hashRandomFile())) - count++ - } -} - -func writeRandomFile() error { - rf, err := os.Create("/random-content-file") - if err != nil { - return fmt.Errorf("while opening file: %w", err) - } - defer rf.Close() - - _, err = io.CopyN(rf, rand.Reader, 1*1024*1024) - if err != nil { - return fmt.Errorf("while copying rand data: %w", err) - } - - return nil -} - -func hashRandomFile() string { - rfBytes, err := os.ReadFile("/random-content-file") - if err != nil { - panic(err) + // The server is the process's reason to live: block on it instead of a + // keep-alive loop, so the process exits when (and only when) it fails. + slog.InfoContext(ctx, "Starting counter server on port 80") + if err := http.ListenAndServe(":80", defaultMux); err != nil { + slog.ErrorContext(ctx, "Error starting server", slog.Any("err", err)) + os.Exit(1) } - - hash := sha256.Sum256(rfBytes) - return base64.RawStdEncoding.EncodeToString(hash[:]) } func getCurrentIP() string { diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index fa8801842..3447a21c8 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -59,6 +59,8 @@ ${EXTERNAL_VOLUME_MOUNTS} snapshotsConfig: onPause: Full onCommit: Data + onResume: + fromData: Golden location: gs://${BUCKET_NAME}/ate-demo-counter/ volumes: - name: data diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 85e732c82..73f8c18cc 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -239,13 +239,15 @@ func TestDurableDirLifecycle(t *testing.T) { }, }, { - // OnGolden data resume: the suspend captures only the durable data - // (the snapshot records plain Data content); the resume combines it - // with the template's golden snapshot per onResume.fromData. The - // golden guest was never called, so its restored memory counter is - // 0 — the counter expectations match ColdBoot, while the file - // counter proves the durable data came from the ACTOR's snapshot - // (the golden's own durable tar would read 0). + // OnGolden data resume, on both runtimes: the suspend captures only + // the durable data (the snapshot records plain Data content); the + // resume combines it with the template's golden snapshot per + // onResume.fromData — micro-VM restores one merged folder, gVisor + // restores the golden split checkpoint with the actor's data as its + // fs/ half. The golden guest was never called, so its restored + // memory counter is 0 — the counter expectations match ColdBoot, + // while the file counter proves the durable data came from the + // ACTOR's snapshot (the golden's own durable data would read 0). name: "onCommit:Data, onPause:Full, onResume.fromData:Golden", tc: actorLifecycleTestCase{ onCommit: v1alpha1.SnapshotScopeData, @@ -256,7 +258,6 @@ func TestDurableDirLifecycle(t *testing.T) { wantMemoryAfterSuspend: 1, wantFileAfterSuspend: 3, wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - microVMOnly: true, }, }, { @@ -273,16 +274,12 @@ func TestDurableDirLifecycle(t *testing.T) { wantMemoryAfterSuspend: 1, wantFileAfterSuspend: 3, wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - microVMOnly: true, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if test.tc.microVMOnly && !isMicroVMEnvironment() { - t.Skipf("Skipping %s: the Golden resume source is micro-VM only", test.name) - } t.Parallel() runActorLifecycleTestCase(t, "durabledir-lifecycle", createActorTemplate, test.tc) }) @@ -338,16 +335,12 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { wantFileAfterSuspend: 3, checkSecondFileCounter: true, wantSnapshotContentScope: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA, - microVMOnly: true, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if test.tc.microVMOnly && !isMicroVMEnvironment() { - t.Skipf("Skipping %s: the Golden resume source is micro-VM only", test.name) - } t.Parallel() runActorLifecycleTestCase(t, "multi-durabledir-lifecycle", createActorTemplateWithTwoDurableDirs, test.tc) }) @@ -412,10 +405,6 @@ type actorLifecycleTestCase struct { // the golden-combine is a restore-time behavior derived from the // template's onResume.fromData source, never part of the snapshot record. wantSnapshotContentScope ateapipb.SnapshotContentScope - - // microVMOnly skips the case outside the micro-VM environment (e.g. - // fromData: Golden is rejected by the CRD CEL rules on gVisor). - microVMOnly bool } func runActorLifecycleTestCase(t *testing.T, prefix string, createTemplate func(context.Context, *testing.T, *e2e.Clients, *e2e.Namespace, v1alpha1.SnapshotScope, v1alpha1.SnapshotScope, v1alpha1.ResumeSource) (*v1alpha1.ActorTemplate, error), tc actorLifecycleTestCase) { diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 24f64fec0..c6061aaba 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -330,8 +330,7 @@ spec: default: {} description: |- OnResume specifies, per snapshot situation, what supplies the guest - state at resume (see OnResumeConfig). "fromData: Golden" requires - sandboxClass "microvm". + state at resume (see OnResumeConfig). properties: fromData: default: ColdBoot @@ -468,11 +467,6 @@ spec: - message: ExternalVolumes are not supported when sandboxClass is 'microvm' rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' - - message: 'onResume.fromData: Golden is not supported when sandboxClass - is ''gvisor''' - rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') - || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) - ? self.snapshotsConfig.onResume.fromData : ''ColdBoot'') != ''Golden''' status: description: status is the observed state of ActorTemplate properties: diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 1a3c9d6f2..8d138b9ac 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -28,9 +28,16 @@ spec: default: true assets: amd64: - gvisor: - url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" - sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + # TEMPORARY (draft): a runsc build with -split-fscheckpoint / + # -split-fsrestore, required by the OnGolden data resume. Switch back to + # the official gvisor release tarball below once a release carries the + # split-checkpoint flags. + runsc: + url: "gs://snapshot-substrate-test-dberkov-gke-dev2/gvisor-split-checkpoint/runsc" + sha256: "7e66fd891b26b9d99dcb9ab531d2c874b20768a7fce2e7477a77e679bbfab1b9" + # gvisor: + # url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + # sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" arm64: gvisor: url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index 17c07b29a..3c939c2c2 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -296,7 +296,7 @@ const ( // ResumeSourceGolden restores the ActorTemplate's golden snapshot (guest // memory + filesystem delta) and serves the snapshot's durable data to // it, so the actor resumes with the golden's warm state over its own - // data. Requires sandboxClass "microvm". + // data. ResumeSourceGolden ResumeSource = "Golden" ) @@ -343,8 +343,7 @@ type SnapshotsConfig struct { OnCommit SnapshotScope `json:"onCommit,omitempty"` // OnResume specifies, per snapshot situation, what supplies the guest - // state at resume (see OnResumeConfig). "fromData: Golden" requires - // sandboxClass "microvm". + // state at resume (see OnResumeConfig). // // +optional // +kubebuilder:default={} @@ -355,7 +354,6 @@ type SnapshotsConfig struct { // // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" -// +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" type ActorTemplateSpec struct { // PauseImage is the container to use as the root sandbox container. // diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 327e9f41f..573108ca9 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -645,22 +645,20 @@ func TestActorTemplateValidation(t *testing.T) { wantErr: true, errMsg: "Unsupported value", }, { - name: "SnapshotsConfig: onResume.fromData=Golden, explicit gvisor (invalid)", + name: "SnapshotsConfig: onResume.fromData=Golden, explicit gvisor", mutate: func(at *ActorTemplate) { at.Spec.SandboxClass = SandboxClassGvisor at.Spec.SnapshotsConfig.OnCommit = SnapshotScopeData at.Spec.SnapshotsConfig.OnResume = OnResumeConfig{FromData: ResumeSourceGolden} }, - wantErr: true, - errMsg: "onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'", + wantErr: false, }, { - name: "SnapshotsConfig: onResume.fromData=Golden, SandboxClass unset (defaults to gvisor, invalid)", + name: "SnapshotsConfig: onResume.fromData=Golden, SandboxClass unset (defaults to gvisor)", mutate: func(at *ActorTemplate) { at.Spec.SnapshotsConfig.OnCommit = SnapshotScopeData at.Spec.SnapshotsConfig.OnResume = OnResumeConfig{FromData: ResumeSourceGolden} }, - wantErr: true, - errMsg: "onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'", + wantErr: false, }, { name: "Volumes: 1 DurableDir mount is valid", mutate: func(at *ActorTemplate) {