Skip to content

gvisor: resume Data snapshots on the golden snapshot (#451, phase 2) #790

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Dmitry Berkovich (dberkov) wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

gvisor: resume Data snapshots on the golden snapshot (#451, phase 2) #790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 78 additions & 11 deletions cmd/atelet/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ func toAteomSnapshotScope(scope ateletpb.SnapshotScope) ateompb.SnapshotScope {

func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error {
localCheckpointPath := filepath.Join(ateompath.LocalCheckpointsDir(req.GetActorUid()), req.GetLocalConfig().GetSnapshotPrefix())
if err := os.MkdirAll(localCheckpointPath, 0o700); err != nil {
if err := ensureParentDirs(localCheckpointPath, rec.SnapshotFiles); err != nil {
return fmt.Errorf("while creating local checkpoint directory: %w", err)
}

Expand Down Expand Up @@ -620,6 +620,15 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
if goldenRec.SandboxClass != sandboxRec.SandboxClass {
return nil, status.Errorf(codes.FailedPrecondition, "golden snapshot sandbox class %q does not match actor snapshot sandbox class %q", goldenRec.SandboxClass, sandboxRec.SandboxClass)
}
// gVisor checkpoint images are runsc-version-coupled: the actor's
// data fs image and the golden's memory image can only be combined
// when both were produced by the same pinned runsc. The micro-VM
// data half is a plain tar, so no such coupling there.
if sandboxRec.SandboxClass == sandboxClassGvisor {
if got, want := gvisorRuntimeAssetSHA(sandboxRec), gvisorRuntimeAssetSHA(goldenRec); got != want {
return nil, status.Errorf(codes.FailedPrecondition, "actor snapshot gVisor runtime (sha256 %s) does not match golden snapshot gVisor runtime (sha256 %s); the snapshots cannot be combined", got, want)
}
}
}

// The record whose pinned binaries run the restored workload: the golden's
Expand All @@ -633,6 +642,24 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
runtimeRec = goldenRec
}

// Where a DATA_ON_GOLDEN restore stages each half, so the restore dir
// ends up looking like a Full snapshot whose durable data is the ACTOR's.
// Micro-VM: one flat folder — the actor's files shadow the golden's
// same-named files (its durable tar replaces the golden's).
// gVisor: the golden's split checkpoint keeps memory files at the top level
// and the durable fs image under fs/; the golden's fs/ contents are skipped
// and the actor's data snapshot (a flat fscheckpoint image set) is staged
// into fs/ instead, overriding it wholesale.
actorDstDir := checkpointDir
var goldenFiles []string
if goldenRec != nil {
goldenFiles = goldenOnlyFiles(sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles)
if sandboxRec.SandboxClass == sandboxClassGvisor {
actorDstDir = filepath.Join(checkpointDir, gvisorSplitFsSubdir)
goldenFiles = filesOutsideDir(goldenRec.SnapshotFiles, gvisorSplitFsSubdir)
}
}

// Download the memory snapshot and prepare the sandbox assets + OCI bundle
// CONCURRENTLY. They are independent — only the final ateom.RestoreWorkload
// needs both — so overlapping the GCS download (~0.5s warm) with the asset
Expand All @@ -650,7 +677,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
if goldenRec == nil {
return fmt.Errorf("no golden snapshot record for a %s restore", req.GetScope())
}
if err := s.downloadCombinedCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), req.GetGoldenSnapshotUriPrefix(), checkpointDir, sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles); err != nil {
if err := s.downloadCombinedCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), actorDstDir, sandboxRec.SnapshotFiles, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenFiles); err != nil {
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError)
}
} else if err := s.downloadExternalCheckpoint(gctx, req.GetExternalConfig().GetSnapshotUriPrefix(), checkpointDir, sandboxRec.SnapshotFiles); err != nil {
Expand All @@ -666,14 +693,14 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
// the golden's from object storage, concurrently.
gLocal, gLocalCtx := errgroup.WithContext(gctx)
gLocal.Go(func() error {
if err := s.copyLocalCheckpoint(gLocalCtx, req.GetLocalConfig().GetSnapshotPrefix(), ateompath.LocalCheckpointsDir(actorUID), checkpointDir, sandboxRec.SnapshotFiles); err != nil {
if err := s.copyLocalCheckpoint(gLocalCtx, req.GetLocalConfig().GetSnapshotPrefix(), ateompath.LocalCheckpointsDir(actorUID), actorDstDir, sandboxRec.SnapshotFiles); err != nil {
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonTerminalFileSystemError)
}
return nil
})
if combineWithGolden {
gLocal.Go(func() error {
if err := s.downloadExternalCheckpoint(gLocalCtx, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenOnlyFiles(sandboxRec.SnapshotFiles, goldenRec.SnapshotFiles)); err != nil {
if err := s.downloadExternalCheckpoint(gLocalCtx, req.GetGoldenSnapshotUriPrefix(), checkpointDir, goldenFiles); err != nil {
return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError)
}
return nil
Expand Down Expand Up @@ -749,6 +776,9 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest)
}

func (s *AteomHerder) copyLocalCheckpoint(ctx context.Context, snapshotPrefix string, srcDir, dstDir string, files []string) error {
if err := ensureParentDirs(dstDir, files); err != nil {
return err
}
for _, fileName := range files {
if ctx.Err() != nil {
return fmt.Errorf("context cancelled: %w", ctx.Err())
Expand Down Expand Up @@ -807,28 +837,65 @@ func goldenOnlyFiles(actorFiles, goldenFiles []string) []string {
return rest
}

// downloadCombinedCheckpoint stages a DATA_ON_GOLDEN restore set into dstDir
// as a single folder: every file of the actor's own snapshot (the durable-dir
// data) plus the golden snapshot's files the actor's set does not shadow, so
// the result looks like a Full snapshot whose durable-dir data is the actor's.
func (s *AteomHerder) downloadCombinedCheckpoint(ctx context.Context, actorPrefix, goldenPrefix, dstDir string, actorFiles, goldenFiles []string) error {
// downloadCombinedCheckpoint stages both halves of a DATA_ON_GOLDEN restore
// concurrently: the actor's own snapshot files (the durable-dir data) into
// actorDstDir and the given golden snapshot files into goldenDstDir. The
// caller decides the layout: one flat folder with the actor's files
// shadowing the golden's (micro-VM), or the actor's files re-rooted into
// the golden checkpoint's fs/ subfolder (gVisor).
func (s *AteomHerder) downloadCombinedCheckpoint(ctx context.Context, actorPrefix, actorDstDir string, actorFiles []string, goldenPrefix, goldenDstDir string, goldenFiles []string) error {
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error {
return s.downloadExternalCheckpoint(gctx, actorPrefix, dstDir, actorFiles)
return s.downloadExternalCheckpoint(gctx, actorPrefix, actorDstDir, actorFiles)
})
g.Go(func() error {
return s.downloadExternalCheckpoint(gctx, goldenPrefix, dstDir, goldenOnlyFiles(actorFiles, goldenFiles))
return s.downloadExternalCheckpoint(gctx, goldenPrefix, goldenDstDir, goldenFiles)
})
return g.Wait()
}

// filesOutsideDir returns the names not under the given top-level subdir. A
// gVisor DATA_ON_GOLDEN restore fetches the golden's memory files (top
// level) but not its fs/ contents, which the actor's data replaces.
func filesOutsideDir(files []string, subdir string) []string {
kept := make([]string, 0, len(files))
for _, f := range files {
if !strings.HasPrefix(f, subdir+"/") {
kept = append(kept, f)
}
}
return kept
}

// ensureParentDirs creates, once each, the distinct parent directories the
// given (possibly subdir-relative)
// snapshot file names need under dstDir.
func ensureParentDirs(dstDir string, files []string) error {
made := make(map[string]struct{}, 2)
for _, f := range files {
dir := filepath.Dir(filepath.Join(dstDir, f))
if _, ok := made[dir]; ok {
continue
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return fmt.Errorf("while creating snapshot subdirectory %q: %w", dir, err)
}
made[dir] = struct{}{}
}
return nil
}

func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUriPrefix string, dstDir string, files []string) error {
prefix := strings.TrimSuffix(snapshotUriPrefix, "/")
if err := ensureParentDirs(dstDir, files); err != nil {
return err
}
g, gCtx := errgroup.WithContext(ctx)
for _, fileName := range files {
fileName := fileName
local := filepath.Join(dstDir, fileName)
g.Go(func() error {
slog.InfoContext(ctx, "@@@@@ [downloadExternalCheckpoint #1]", slog.String("file", prefix+"/"+fileName+".zstd"), slog.String("local", local))
if err := ategcs.FetchLocalFileFromGCSWithZstd(gCtx, s.gcsClient, prefix+"/"+fileName+".zstd", local); err != nil {
return fmt.Errorf("while downloading %s from GCS: %w", fileName, err)
}
Expand Down
84 changes: 75 additions & 9 deletions cmd/atelet/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,10 +786,11 @@ func (m mapObjectStorage) GetObject(_ context.Context, bucket, object string) (i

func (mapObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil }

// TestDownloadCombinedCheckpoint verifies a DataOnGolden restore stages one
// folder holding the actor snapshot's durable-dir tar and the golden
// snapshot's remaining files — and that the golden's own durable-dir tar is
// the one that loses the name collision.
// TestDownloadCombinedCheckpoint verifies the micro-VM DataOnGolden layout:
// one flat folder holding the actor snapshot's durable-dir tar and the golden
// snapshot's remaining files — the caller shadows the golden's own
// durable-dir tar out of the golden file list (goldenOnlyFiles), so the
// actor's tar wins the name collision.
func TestDownloadCombinedCheckpoint(t *testing.T) {
zstdBytes := func(t *testing.T, s string) []byte {
t.Helper()
Expand All @@ -816,12 +817,11 @@ func TestDownloadCombinedCheckpoint(t *testing.T) {
s := &AteomHerder{gcsClient: store}

dstDir := t.TempDir()
actorFiles := []string{"durable-dir.tar"}
goldenFiles := []string{"config.json", "memory-ranges", "durable-dir.tar"}
err := s.downloadCombinedCheckpoint(context.Background(),
"gs://bucket/actors/1/snapshots/2/",
"gs://bucket/ate-golden/snapshots/1/",
dstDir,
[]string{"durable-dir.tar"},
[]string{"config.json", "memory-ranges", "durable-dir.tar"})
"gs://bucket/actors/1/snapshots/2/", dstDir, actorFiles,
"gs://bucket/ate-golden/snapshots/1/", dstDir, goldenOnlyFiles(actorFiles, goldenFiles))
if err != nil {
t.Fatalf("downloadCombinedCheckpoint: %v", err)
}
Expand Down Expand Up @@ -988,3 +988,69 @@ func TestDrainOnShutdownForceStopsAfterTimeout(t *testing.T) {
t.Fatal("readiness should be not-ready after drain")
}
}

// TestDownloadCombinedCheckpointGvisorFsOverride verifies the gVisor
// DataOnGolden layout: the golden split checkpoint's memory files land at
// the restore dir's top level (its own fs/ contents are excluded by
// filesOutsideDir), while the actor's flat fscheckpoint image set is
// re-rooted into the fs/ subfolder, overriding it wholesale — and the
// subfolder is created on demand.
func TestDownloadCombinedCheckpointGvisorFsOverride(t *testing.T) {
zstdBytes := func(t *testing.T, s string) []byte {
t.Helper()
var buf bytes.Buffer
zw, err := zstd.NewWriter(&buf)
if err != nil {
t.Fatalf("zstd.NewWriter: %v", err)
}
if _, err := zw.Write([]byte(s)); err != nil {
t.Fatalf("zstd write: %v", err)
}
if err := zw.Close(); err != nil {
t.Fatalf("zstd close: %v", err)
}
return buf.Bytes()
}

store := mapObjectStorage{objects: map[string][]byte{
"bucket/actors/1/snapshots/2/multitar.img.zstd": zstdBytes(t, "actor data fs image"),
"bucket/ate-golden/snapshots/1/checkpoint.img.zstd": zstdBytes(t, "golden memory"),
"bucket/ate-golden/snapshots/1/pages.img.zstd": zstdBytes(t, "golden pages"),
"bucket/ate-golden/snapshots/1/fs/multitar.img.zstd": zstdBytes(t, "golden fs image (must not be downloaded)"),
}}
s := &AteomHerder{gcsClient: store}

dstDir := t.TempDir()
actorFiles := []string{"multitar.img"}
goldenFiles := []string{"checkpoint.img", "fs/multitar.img", "pages.img"}
err := s.downloadCombinedCheckpoint(context.Background(),
"gs://bucket/actors/1/snapshots/2/", filepath.Join(dstDir, "fs"), actorFiles,
"gs://bucket/ate-golden/snapshots/1/", dstDir, filesOutsideDir(goldenFiles, "fs"))
if err != nil {
t.Fatalf("downloadCombinedCheckpoint: %v", err)
}

for path, content := range map[string]string{
filepath.Join(dstDir, "checkpoint.img"): "golden memory",
filepath.Join(dstDir, "pages.img"): "golden pages",
filepath.Join(dstDir, "fs", "multitar.img"): "actor data fs image",
} {
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s): %v", path, err)
}
if string(got) != content {
t.Errorf("%s content = %q, want %q", path, got, content)
}
}
}

// TestFilesOutsideDir verifies the golden-file filter for the gVisor
// fs-override layout, including that a prefix-similar name (fsx) survives.
func TestFilesOutsideDir(t *testing.T) {
got := filesOutsideDir([]string{"checkpoint.img", "fs/multitar.img", "fs/pages.img", "fsx.img"}, "fs")
want := []string{"checkpoint.img", "fsx.img"}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("filesOutsideDir diff (-want +got):\n%s", diff)
}
}
25 changes: 25 additions & 0 deletions cmd/atelet/sandbox_assets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ const sandboxManifestName = "manifest.json"
var maxAssetBytes int64 = 8 << 30

const (
// sandboxClassGvisor mirrors the gVisor SandboxClass value recorded in a
// manifest's sandboxClass field (see atev1alpha1.SandboxClassGvisor;
// atelet does not depend on the CRD API package). Not to be confused
// with gvisorAssetName below: the class names the runtime family, the
// asset name keys one entry of the class's asset map, and the two only
// coincidentally share the value "gvisor".
sandboxClassGvisor = "gvisor"

// gvisorAssetName is the gVisor release tarball asset (gvisor.tar.bz2). The
// tarball carries `runsc` together with the `gvisor-bin/` helper binaries,
// so it is extracted into a content-addressed directory rather than as a
Expand All @@ -58,6 +66,12 @@ const (

// runscAssetName is the legacy single-binary gVisor asset.
runscAssetName = "runsc"

// gvisorSplitFsSubdir is the subfolder of a gVisor split checkpoint
// (`runsc checkpoint -split-fscheckpoint`) holding the durable-dir fs
// image; the memory snapshot files sit at the checkpoint's top level, and
// `runsc restore -split-fsrestore` reads the fs half from this subfolder.
gvisorSplitFsSubdir = "fs"
)

// assetEntry is one content-addressed sandbox asset (url + sha256).
Expand Down Expand Up @@ -142,6 +156,17 @@ func (s *AteomHerder) ensureSandboxAssets(ctx context.Context, rec *sandboxAsset
return paths, nil
}

// gvisorRuntimeAssetSHA returns the sha256 pinning a record's gVisor runtime:
// the release tarball asset, or the legacy bare runsc binary for records
// written before the tarball mechanism. Used to require that the two halves
// of a DATA_ON_GOLDEN combine were produced by the same runsc.
func gvisorRuntimeAssetSHA(rec *sandboxAssetsRecord) string {
if a, ok := rec.Assets[gvisorAssetName]; ok {
return a.SHA256
}
return rec.Assets[runscAssetName].SHA256
}

// runscPathFor returns the local path of the gVisor `runsc` binary from a
// fetched asset-path map, or "" if the runtime has none (e.g. micro-VM).
func runscPathFor(paths map[string]string) string {
Expand Down
46 changes: 34 additions & 12 deletions cmd/ateom-gvisor/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"log/slog"
"net"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -439,15 +441,22 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
// listSnapshotFiles returns the (relative) names of regular files directly under
// dir, which atelet ships to object storage as the snapshot.
func listSnapshotFiles(dir string) ([]string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
var files []string
for _, e := range entries {
if e.Type().IsRegular() {
files = append(files, e.Name())
if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.Type().IsRegular() {
return nil
}
rel, err := filepath.Rel(dir, path)
if err != nil {
return err
}
files = append(files, rel)
return nil
}); err != nil {
return nil, err
}
sort.Strings(files)
return files, nil
Expand Down Expand Up @@ -529,18 +538,30 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
return nil, fmt.Errorf("while composing pause rootfs: %w", err)
}

fsRestoreArgs := func(dir string) []string {
if _, err := os.Stat(filepath.Join(dir, "fs")); err == nil {
return []string{"--fs-restore-image-path", dir + "/fs"}
}
return nil
}

switch req.GetScope() {
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA:
// Create and restore pause container
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", []string{"--fs-restore-image-path", checkpointDir}); err != nil {
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", fsRestoreArgs(checkpointDir)); err != nil {
return nil, fmt.Errorf("while creating pause container: %w", err)
}
if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil {
return nil, fmt.Errorf("while starting pause container: %w", err)
}
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
// Create and restore pause container
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN:
// DATA_ON_GOLDEN restores exactly like FULL: atelet staged the golden
// checkpoint's memory files at the restore dir's top level and the
// ACTOR's data fs image in its fs/ subfolder, which is where
// `runsc restore -split-fsrestore` reads the fs half from anyway.
// Create and restore pause container.
if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", fsRestoreArgs(checkpointDir)); err != nil {
return nil, fmt.Errorf("while creating pause container: %w", err)
}
if err := rcmd.cmdRestore(ctx, os.Stdout, "pause", checkpointDir); err != nil {
Expand Down Expand Up @@ -569,7 +590,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
if err := rcmd.cmdStart(ctx, pw, ac.GetName()); err != nil {
return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err)
}
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN:
if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil {
return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err)
}
Expand Down
Loading
Loading