diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 61c38ff8..335da92f 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -638,6 +638,7 @@ func (s *ApiService) ForkInstance(ctx context.Context, request oapi.ForkInstance Name: request.Body.Name, FromRunning: request.Body.FromRunning != nil && *request.Body.FromRunning, TargetState: targetState, + ShareMemory: request.Body.ShareMemory != nil && *request.Body.ShareMemory, }) if err != nil { switch { @@ -1062,6 +1063,8 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { ExitCode: inst.ExitCode, HasSnapshot: lo.ToPtr(inst.HasSnapshot), Hypervisor: &hvType, + ForkCount: lo.ToPtr(inst.ForkCount), + MemLocked: lo.ToPtr(inst.MemLocked), } if b, err := json.Marshal(networkPayload); err == nil { diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 6dc6eecc..0d389f8d 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -11,10 +11,28 @@ import ( var ErrSparseCopyUnsupported = errors.New("sparse copy unsupported") +// CopyOptions tunes CopyGuestDirectory behavior. The zero value reproduces +// the original full-copy semantics; callers can opt into skipping specific +// paths when the consumer arranges its own substitute (e.g. a symlink to a +// template-shared mem-file). +type CopyOptions struct { + // SkipRelPaths lists relative paths under srcDir that should not be + // materialized in dstDir. Comparison is exact and uses forward-slash + // separators on all platforms. + SkipRelPaths []string +} + // CopyGuestDirectory recursively copies a guest directory to a new destination. // Regular files are copied using sparse extent copy only (SEEK_DATA/SEEK_HOLE). // Runtime sockets and logs are skipped because they are host-runtime artifacts. func CopyGuestDirectory(srcDir, dstDir string) error { + return CopyGuestDirectoryWithOptions(srcDir, dstDir, CopyOptions{}) +} + +// CopyGuestDirectoryWithOptions is the option-taking variant of +// CopyGuestDirectory. Use this when forking with template-shared assets, so +// the caller can install a symlink in place of a heavy copied file. +func CopyGuestDirectoryWithOptions(srcDir, dstDir string, opts CopyOptions) error { srcInfo, err := os.Stat(srcDir) if err != nil { return fmt.Errorf("stat source directory: %w", err) @@ -27,6 +45,11 @@ func CopyGuestDirectory(srcDir, dstDir string) error { return fmt.Errorf("create destination directory: %w", err) } + skipSet := make(map[string]struct{}, len(opts.SkipRelPaths)) + for _, p := range opts.SkipRelPaths { + skipSet[filepath.ToSlash(p)] = struct{}{} + } + return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -39,6 +62,12 @@ func CopyGuestDirectory(srcDir, dstDir string) error { if relPath == "." { return nil } + if _, skip := skipSet[filepath.ToSlash(relPath)]; skip { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } if d.IsDir() && shouldSkipDirectory(relPath) { return filepath.SkipDir } diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go index c71f6c4e..56fb6caf 100644 --- a/lib/forkvm/copy_test.go +++ b/lib/forkvm/copy_test.go @@ -44,6 +44,25 @@ func TestCopyGuestDirectory(t *testing.T) { assert.Equal(t, "metadata.json", linkTarget) } +func TestCopyGuestDirectory_SkipRelPaths(t *testing.T) { + src := filepath.Join(t.TempDir(), "src") + dst := filepath.Join(t.TempDir(), "dst") + + require.NoError(t, os.MkdirAll(filepath.Join(src, "snapshots", "snapshot-latest"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory"), []byte("the heavy mem-file"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "state"), []byte("device state"), 0644)) + + err := CopyGuestDirectoryWithOptions(src, dst, CopyOptions{ + SkipRelPaths: []string{"snapshots/snapshot-latest/memory"}, + }) + require.NoError(t, err) + + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory")) + assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json")) + assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "state")) +} + func TestCopyGuestDirectory_DoesNotSkipTmpSuffixedDirectories(t *testing.T) { src := filepath.Join(t.TempDir(), "src") dst := filepath.Join(t.TempDir(), "dst") diff --git a/lib/hypervisor/firecracker/config.go b/lib/hypervisor/firecracker/config.go index 9576f2ca..192103ea 100644 --- a/lib/hypervisor/firecracker/config.go +++ b/lib/hypervisor/firecracker/config.go @@ -75,12 +75,21 @@ type snapshotCreateParams struct { type snapshotLoadParams struct { MemFilePath string `json:"mem_file_path,omitempty"` + MemBackend *memBackend `json:"mem_backend,omitempty"` SnapshotPath string `json:"snapshot_path"` EnableDiffSnapshots bool `json:"enable_diff_snapshots,omitempty"` ResumeVM bool `json:"resume_vm,omitempty"` NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` } +// memBackend selects how firecracker materializes guest memory during +// restore. backend_type "Uffd" hands page-fault handling off to a +// userfaultfd page server reachable at backend_path (Unix domain socket). +type memBackend struct { + BackendType string `json:"backend_type"` + BackendPath string `json:"backend_path"` +} + type networkOverride struct { IfaceID string `json:"iface_id"` HostDevName string `json:"host_dev_name"` @@ -103,6 +112,11 @@ type instanceInfo struct { type restoreMetadata struct { NetworkOverrides []networkOverride `json:"network_overrides,omitempty"` SnapshotSourceDataDir string `json:"snapshot_source_data_dir,omitempty"` + // UffdSocketPath, when non-empty, makes loadSnapshot send a Uffd + // mem_backend pointing at the page server instead of letting + // firecracker mmap the mem-file directly. PrepareFork records it + // per fork so RestoreVM can pick it up after a hypeman restart. + UffdSocketPath string `json:"uffd_socket_path,omitempty"` } func toBootSource(cfg hypervisor.VMConfig) bootSource { @@ -212,14 +226,25 @@ func toSnapshotCreateParams(snapshotDir string) snapshotCreateParams { } } -func toSnapshotLoadParams(snapshotDir string, networkOverrides []networkOverride) snapshotLoadParams { - return snapshotLoadParams{ - MemFilePath: snapshotMemoryPath(snapshotDir), +func toSnapshotLoadParams(snapshotDir string, networkOverrides []networkOverride, uffdSocketPath string) snapshotLoadParams { + params := snapshotLoadParams{ SnapshotPath: snapshotStatePath(snapshotDir), EnableDiffSnapshots: true, ResumeVM: false, NetworkOverrides: networkOverrides, } + if uffdSocketPath != "" { + // Firecracker rejects load requests that set both mem_file_path + // and a uffd backend. The page server takes the file path through + // its own configuration, so we drop it from the request. + params.MemBackend = &memBackend{ + BackendType: "Uffd", + BackendPath: uffdSocketPath, + } + } else { + params.MemFilePath = snapshotMemoryPath(snapshotDir) + } + return params } func snapshotStatePath(snapshotDir string) string { diff --git a/lib/hypervisor/firecracker/config_test.go b/lib/hypervisor/firecracker/config_test.go index 6e912ee7..7e49b4f2 100644 --- a/lib/hypervisor/firecracker/config_test.go +++ b/lib/hypervisor/firecracker/config_test.go @@ -82,12 +82,19 @@ func TestSnapshotParamPaths(t *testing.T) { load := toSnapshotLoadParams("/tmp/snapshot-latest", []networkOverride{ {IfaceID: "eth0", HostDevName: "hype-abc123"}, - }) + }, "") assert.Equal(t, "/tmp/snapshot-latest/state", load.SnapshotPath) assert.Equal(t, "/tmp/snapshot-latest/memory", load.MemFilePath) + assert.Nil(t, load.MemBackend) assert.True(t, load.EnableDiffSnapshots) assert.False(t, load.ResumeVM) require.Len(t, load.NetworkOverrides, 1) + + loadUffd := toSnapshotLoadParams("/tmp/snapshot-latest", nil, "/run/uffd/abc.sock") + assert.Equal(t, "", loadUffd.MemFilePath, "mem_file_path must be empty when a uffd backend is set") + require.NotNil(t, loadUffd.MemBackend) + assert.Equal(t, "Uffd", loadUffd.MemBackend.BackendType) + assert.Equal(t, "/run/uffd/abc.sock", loadUffd.MemBackend.BackendPath) } func TestToBalloonConfig(t *testing.T) { diff --git a/lib/hypervisor/firecracker/firecracker.go b/lib/hypervisor/firecracker/firecracker.go index e22e42f3..e3bf435e 100644 --- a/lib/hypervisor/firecracker/firecracker.go +++ b/lib/hypervisor/firecracker/firecracker.go @@ -223,8 +223,8 @@ func (f *Firecracker) instanceStart(ctx context.Context) error { return f.postAction(ctx, "InstanceStart") } -func (f *Firecracker) loadSnapshot(ctx context.Context, snapshotDir string, networkOverrides []networkOverride) error { - params := toSnapshotLoadParams(snapshotDir, networkOverrides) +func (f *Firecracker) loadSnapshot(ctx context.Context, snapshotDir string, networkOverrides []networkOverride, uffdSocketPath string) error { + params := toSnapshotLoadParams(snapshotDir, networkOverrides, uffdSocketPath) if _, err := f.do(ctx, http.MethodPut, "/snapshot/load", params, http.StatusNoContent); err != nil { return err } diff --git a/lib/hypervisor/firecracker/fork.go b/lib/hypervisor/firecracker/fork.go index 81936929..3cb8049e 100644 --- a/lib/hypervisor/firecracker/fork.go +++ b/lib/hypervisor/firecracker/fork.go @@ -53,6 +53,10 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq changed = true } } + if meta.UffdSocketPath != req.UffdSocketPath { + meta.UffdSocketPath = req.UffdSocketPath + changed = true + } if changed { if err := saveRestoreMetadataState(instanceDir, meta); err != nil { diff --git a/lib/hypervisor/firecracker/process.go b/lib/hypervisor/firecracker/process.go index 371e1f2e..fbe6cbea 100644 --- a/lib/hypervisor/firecracker/process.go +++ b/lib/hypervisor/firecracker/process.go @@ -119,7 +119,7 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, snapshotSourceAliasMu.Lock() defer snapshotSourceAliasMu.Unlock() return withSnapshotSourceDirAlias(meta, filepath.Dir(socketPath), func() error { - return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides) + return hv.loadSnapshot(ctx, snapshotPath, meta.NetworkOverrides, meta.UffdSocketPath) }) }() if err != nil { diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index a3e5d01f..e593a9bf 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -146,6 +146,12 @@ type ForkPrepareRequest struct { SerialLogPath string Network *ForkNetworkConfig + + // UffdSocketPath is set when the fork should restore from a userfaultfd + // page-server socket instead of mmap'ing its mem-file directly. The + // hypervisor records this so RestoreVM can attach a uffd memory backend + // in the snapshot/load request. Empty means use the default mmap path. + UffdSocketPath string } // ForkPrepareResult describes which optional fork rewrites were actually applied. diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 283e0b19..6011c057 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -35,6 +35,18 @@ func (m *manager) deleteInstance( stored := &meta.StoredMetadata log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State) + // If this instance was promoted to a template parent, refuse to delete + // it while live forks reference it. Removing the registry entry now + // (instead of after the data wipe) gives us a single transactional + // "in-use" check via templates.ErrInUse. + if stored.IsTemplate && stored.TemplateID != "" && m.templateRegistry != nil { + if err := m.templateRegistry.Delete(ctx, stored.TemplateID); err != nil { + return fmt.Errorf("delete template registry entry for instance %s: %w", id, err) + } + stored.IsTemplate = false + stored.TemplateID = "" + } + target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) if err != nil { return fmt.Errorf("wait for instance compression to stop: %w", err) @@ -136,6 +148,18 @@ func (m *manager) deleteInstance( return fmt.Errorf("delete instance data: %w", err) } + // 9. If this instance was a fork of a template, drop the template's + // fork refcount so the template can eventually be deleted, and + // detach it from the uffd page server if one is running. + if stored.ForkOfTemplate != "" { + m.dropTemplateForkRefcount(ctx, stored.ForkOfTemplate) + if m.uffd != nil { + if err := m.uffd.releaseUffdForFork(stored.ForkOfTemplate, id); err != nil { + log.WarnContext(ctx, "failed to release uffd page server for fork", "instance_id", id, "template_id", stored.ForkOfTemplate, "error", err) + } + } + } + log.InfoContext(ctx, "instance deleted successfully", "instance_id", id) return nil } diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 25973930..3b115d67 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" "time" @@ -544,3 +545,111 @@ func TestFirecrackerSnapshotFeature(t *testing.T) { forkName: "fc-snapshot-fork", }) } + +// TestFirecrackerForkFromTemplate exercises the full template-driven fork +// path: standby a firecracker source, promote it to a template, fork off it, +// and assert the fork (a) reaches Running, (b) has its mem-file hardlinked +// to the template's snapshot mem-file (the fan-out optimisation), (c) bumped +// the template's fork refcount, (d) registered with the per-template uffd +// page server, and (e) on delete, drops the refcount and detaches from uffd. +func TestFirecrackerForkFromTemplate(t *testing.T) { + t.Parallel() + requireFirecrackerIntegrationPrereqs(t) + + mgr, tmpDir := setupTestManagerForFirecracker(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, mgr.networkManager.Initialize(ctx, nil)) + + source, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: "fc-tpl-src", + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 256 * 1024 * 1024, + OverlaySize: 5 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + Hypervisor: hypervisor.TypeFirecracker, + }) + require.NoError(t, err) + source, err = waitForInstanceState(ctx, mgr, source.Id, StateRunning, integrationTestTimeout(20*time.Second)) + require.NoError(t, err) + sourceID := source.Id + t.Cleanup(func() { _ = mgr.DeleteInstance(context.Background(), sourceID) }) + + // Standby is a precondition for promotion. + source, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateStandby, source.State) + require.True(t, source.HasSnapshot) + + tpl, err := mgr.promoteToTemplate(ctx, sourceID, PromoteToTemplateRequest{Name: "fc-tpl-e2e"}) + require.NoError(t, err) + require.NotNil(t, tpl) + require.Equal(t, sourceID, tpl.SourceInstanceID) + require.Equal(t, hypervisor.TypeFirecracker, tpl.HypervisorType) + require.Equal(t, 0, tpl.ForkCount) + + // Fork from the template (no source instance id passed). + forked, err := mgr.ForkInstance(ctx, "", ForkInstanceRequest{ + Name: "fc-tpl-fork", + TemplateID: tpl.ID, + TargetState: StateRunning, + }) + require.NoError(t, err) + forked, err = waitForInstanceState(ctx, mgr, forked.Id, StateRunning, integrationTestTimeout(30*time.Second)) + require.NoError(t, err) + require.Equal(t, StateRunning, forked.State) + forkID := forked.Id + deletedFork := false + t.Cleanup(func() { + if !deletedFork { + _ = mgr.DeleteInstance(context.Background(), forkID) + } + }) + + // (b) The fork's mem-file must share the source's inode (hardlink), not + // be a copy. We can't compare paths because the link is by inode; we + // compare st_ino + st_dev between the two instances' mem-files. + // + // Firecracker retains the post-restore snapshot dir as snapshot-base + // (see restoreRetainedSnapshotBase), so after the Standby -> Running + // transition the hardlink lives under snapshot-base/, not snapshot-latest/. + // Hardlinks survive the rename because they bind to the inode. + forkMemPath := filepath.Join(p.InstanceSnapshotBase(forkID), templateSharedMemFileName) + srcMemPath := filepath.Join(p.InstanceSnapshotLatest(sourceID), templateSharedMemFileName) + forkInfo, err := os.Stat(forkMemPath) + require.NoError(t, err, "fork mem-file should exist at snapshot-base/memory after restore") + assert.True(t, forkInfo.Mode().IsRegular(), "fork mem-file should be a regular file (hardlink), not a symlink") + srcInfo, err := os.Stat(srcMemPath) + require.NoError(t, err) + forkSys := forkInfo.Sys().(*syscall.Stat_t) + srcSys := srcInfo.Sys().(*syscall.Stat_t) + assert.Equal(t, srcSys.Ino, forkSys.Ino, "fork mem-file should share the source's inode (hardlink, not copy)") + assert.Equal(t, srcSys.Dev, forkSys.Dev, "fork mem-file should be on the same filesystem as source") + + // (c) Refcount on the template must be bumped to 1. + tplAfterFork, err := mgr.getTemplate(ctx, tpl.ID) + require.NoError(t, err) + assert.Equal(t, 1, tplAfterFork.ForkCount, "template fork refcount should be 1 after one fork") + + // (d) The per-template uffd page server should be tracking this fork. + require.NotNil(t, mgr.uffd) + assert.True(t, mgr.uffd.hasFork(tpl.ID, forkID), "uffd tracker should report fork as registered against its template") + + // Deleting the fork drops the refcount and detaches from uffd. + require.NoError(t, mgr.DeleteInstance(ctx, forkID)) + deletedFork = true + + tplAfterDelete, err := mgr.getTemplate(ctx, tpl.ID) + require.NoError(t, err) + assert.Equal(t, 0, tplAfterDelete.ForkCount, "template fork refcount should drop back to 0") + assert.False(t, mgr.uffd.hasFork(tpl.ID, forkID), "uffd tracker should no longer track the deleted fork") +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 4ce7ee6e..ac28e371 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/templates" "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "gvisor.dev/gvisor/pkg/cleanup" @@ -36,11 +37,30 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, "", err } + if req.ShareMemory { + shared, err := m.ensureShareMemoryTemplate(ctx, id) + if err != nil { + return nil, "", err + } + req.TemplateID = shared.ID + id = "" + } + resolvedID, tpl, err := m.resolveForkFromTemplateRequest(ctx, id, req) + if err != nil { + return nil, "", err + } + id = resolvedID + meta, err := m.loadMetadata(id) if err != nil { return nil, "", err } source := m.toInstance(ctx, meta) + if tpl != nil { + if err := validateForkResolvedFromTemplate(tpl, source.HypervisorType); err != nil { + return nil, "", err + } + } targetState, err := resolveForkTargetState(req.TargetState, source.State) if err != nil { return nil, "", err @@ -65,7 +85,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR return nil, "", fmt.Errorf("standby source instance: %w", err) } - forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true) + forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true, tpl) if forkErr == nil { if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil { forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err) @@ -104,7 +124,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR } return forked, targetState, nil case StateStopped, StateStandby: - forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false) + forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false, tpl) if err != nil { return nil, "", err } @@ -192,7 +212,7 @@ func generateForkSourceVsockCID(sourceID, forkID string, current int64) int64 { return cid } -func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool) (*Instance, error) { +func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool, tpl *templates.Template) (*Instance, error) { log := logger.FromContext(ctx) meta, err := m.loadMetadata(id) @@ -202,6 +222,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin source := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if tpl != nil && !stored.IsTemplate { + return nil, fmt.Errorf("%w: template %s source instance %s is not flagged as a template parent", ErrInvalidState, tpl.ID, id) + } switch source.State { case StateStopped, StateStandby: @@ -255,12 +278,21 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin } } - if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil { + copyOpts := forkvm.CopyOptions{} + if tpl != nil { + copyOpts.SkipRelPaths = []string{templateSharedMemFileRelPath} + } + if err := forkvm.CopyGuestDirectoryWithOptions(srcDir, dstDir, copyOpts); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) } return nil, fmt.Errorf("clone guest directory: %w", err) } + if tpl != nil { + if err := m.installForkSharedMemFile(dstDir, tpl); err != nil { + return nil, fmt.Errorf("install shared mem-file: %w", err) + } + } starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { @@ -280,6 +312,15 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(forkMeta.HypervisorType)) forkMeta.ExitCode = nil forkMeta.ExitMessage = "" + // Forks of a template carry the template id but never inherit the + // IsTemplate flag — they are working copies. + forkMeta.IsTemplate = false + forkMeta.TemplateID = "" + if tpl != nil { + forkMeta.ForkOfTemplate = tpl.ID + } else { + forkMeta.ForkOfTemplate = stored.ForkOfTemplate + } // Keep the original CID for snapshot-based forks. // Rewriting CID in restored memory snapshots is not reliable across @@ -303,6 +344,15 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin if forkMeta.NetworkEnabled { netCfg = &hypervisor.ForkNetworkConfig{TAPDevice: network.GenerateTAPName(forkID)} } + uffdSocketPath, err := m.acquireForkUffdIfApplicable(ctx, tpl, forkID, stored.HypervisorType) + if err != nil { + return nil, fmt.Errorf("attach uffd page server: %w", err) + } + if uffdSocketPath != "" { + cu.Add(func() { + _ = m.uffd.releaseUffdForFork(tpl.ID, forkID) + }) + } if _, err := starter.PrepareFork(ctx, hypervisor.ForkPrepareRequest{ SnapshotConfigPath: snapshotConfigPath, SourceDataDir: stored.DataDir, @@ -311,6 +361,7 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin VsockSocket: forkMeta.VsockSocket, SerialLogPath: m.paths.InstanceAppLog(forkID), Network: netCfg, + UffdSocketPath: uffdSocketPath, }); err != nil { if errors.Is(err, hypervisor.ErrNotSupported) { return nil, fmt.Errorf("%w: fork is not supported for hypervisor %s", ErrNotSupported, stored.HypervisorType) @@ -324,6 +375,14 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, fmt.Errorf("save fork metadata: %w", err) } + if tpl != nil { + // Bumped before cu.Release so a refcount failure leaves no orphan + // fork directory (deferred cu.Clean removes the data dir + metadata). + if err := m.bumpTemplateForkRefcount(ctx, tpl); err != nil { + return nil, fmt.Errorf("record template fork refcount: %w", err) + } + } + cu.Release() forked := m.toInstance(ctx, newMeta) log.InfoContext(ctx, "instance forked successfully", @@ -355,6 +414,14 @@ func validateForkRequest(req ForkInstanceRequest) error { if req.TargetState != "" && req.TargetState != StateStopped && req.TargetState != StateStandby && req.TargetState != StateRunning { return fmt.Errorf("%w: invalid fork target state %q (must be one of %s, %s, %s)", ErrInvalidRequest, req.TargetState, StateStopped, StateStandby, StateRunning) } + if req.ShareMemory { + if req.TemplateID != "" { + return fmt.Errorf("%w: share_memory cannot be combined with template_id", ErrInvalidRequest) + } + if req.FromRunning { + return fmt.Errorf("%w: share_memory requires the source to already be in Standby; from_running=true would re-restore the source after locking", ErrInvalidRequest) + } + } return nil } diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index f73e892b..d983c1bc 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -265,7 +265,7 @@ func TestForkInstanceFromStandbyCancelsCompressionJobAndCopiesRawMemory(t *testi forked, err := manager.forkInstanceFromStoppedOrStandby(ctx, sourceID, ForkInstanceRequest{ Name: "fork-standby-compressed-copy", TargetState: StateStopped, - }, true) + }, true, nil) require.NoError(t, err) require.NotNil(t, forked) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index c52add78..629adb2e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/templates" "github.com/kernel/hypeman/lib/volumes" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" @@ -147,6 +148,17 @@ type manager struct { vmStarters map[hypervisor.Type]hypervisor.VMStarter defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request guestMemoryPolicy guestmemory.Policy + + // Template registry. Owned by the manager because template lifecycle + // is coupled to instance lifecycle (promotion + refcount on + // fork/delete). Constructed lazily so existing managers without + // template support keep working unchanged. + templateRegistry templates.Registry + + // uffd is the per-template userfaultfd page-server tracker. nil on + // non-Linux hosts; on Linux it is started lazily for forks that + // resolve to a template and torn down once no forks remain. + uffd *uffdTracker } // platformStarters is populated by platform-specific init functions. @@ -201,6 +213,8 @@ func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemMan compressionJobs: make(map[string]*compressionJob), nativeCodecPaths: make(map[string]string), lifecycleEvents: newLifecycleSubscribersWithBufferSize(managerConfig.LifecycleEventBufferSize), + templateRegistry: templates.NewFileRegistry(p.TemplatesDir()), + uffd: newUffdTracker(), } m.deleteSnapshotFn = m.deleteSnapshot @@ -373,7 +387,13 @@ func (m *manager) DeleteSnapshot(ctx context.Context, snapshotID string) error { // ForkInstance creates a forked copy of an instance. func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) { - lock := m.getInstanceLock(id) + // Resolve TemplateID outside the lock so we hold the source instance + // lock — not an empty string lock — when forking from a template. + resolvedID, _, err := m.resolveForkFromTemplateRequest(ctx, id, req) + if err != nil { + return nil, err + } + lock := m.getInstanceLock(resolvedID) lock.Lock() forked, targetState, err := m.forkInstance(ctx, id, req) lock.Unlock() diff --git a/lib/instances/query.go b/lib/instances/query.go index 699fa35f..c2043068 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -356,6 +356,7 @@ func (m *manager) toInstanceWithStateDerivation(ctx context.Context, meta *metad BootMarkersHydrated: result.BootMarkersHydrated, } refreshHypervisorPID(&inst.StoredMetadata, result.State) + hydrateForkLockState(ctx, m.templateRegistry, &inst) // If VM is stopped and exit info isn't persisted yet, populate in-memory // from the serial console log. This is read-only -- no metadata writes. diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 268f769b..3957e97f 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -57,6 +57,10 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "no snapshot available", "instance_id", id) return nil, fmt.Errorf("no snapshot available for instance %s", id) } + if err := m.templateGuard(stored, "restore"); err != nil { + log.ErrorContext(ctx, "refusing to restore template instance", "instance_id", id, "template_id", stored.TemplateID) + return nil, err + } // 2b. Validate aggregate resource limits before allocating resources (if configured) reservedResources := false diff --git a/lib/instances/share_memory_test.go b/lib/instances/share_memory_test.go new file mode 100644 index 00000000..56b03271 --- /dev/null +++ b/lib/instances/share_memory_test.go @@ -0,0 +1,169 @@ +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateForkRequest_ShareMemoryConflicts(t *testing.T) { + t.Parallel() + + t.Run("share_memory with template_id is rejected", func(t *testing.T) { + err := validateForkRequest(ForkInstanceRequest{ + Name: "fork-bad-combo", + ShareMemory: true, + TemplateID: "tpl-123", + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + }) + + t.Run("share_memory with from_running is rejected", func(t *testing.T) { + err := validateForkRequest(ForkInstanceRequest{ + Name: "fork-bad-combo", + ShareMemory: true, + FromRunning: true, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + }) + + t.Run("share_memory alone is allowed", func(t *testing.T) { + err := validateForkRequest(ForkInstanceRequest{ + Name: "fork-ok", + ShareMemory: true, + }) + require.NoError(t, err) + }) +} + +// stagedStandbySource creates a metadata + fake snapshot directory for an +// instance so toInstance reports State=Standby without involving any real +// hypervisor. Returns the source instance ID. +func stagedStandbySource(t *testing.T, mgr *manager, name string) string { + t.Helper() + id := name + require.NoError(t, mgr.ensureDirectories(id)) + + dataDir := mgr.paths.InstanceDir(id) + snapDir := filepath.Join(dataDir, "snapshots", "snapshot-latest") + require.NoError(t, os.MkdirAll(snapDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(snapDir, "memory"), []byte("fake-mem"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(snapDir, "config.json"), []byte("{}"), 0o644)) + + now := time.Now() + meta := &metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeFirecracker, + HypervisorVersion: "test", + // Intentionally no SocketPath so deriveState falls through to the + // snapshot check and reports Standby. + DataDir: dataDir, + VsockCID: 44, + VsockSocket: mgr.paths.InstanceVsockSocket(id), + }} + require.NoError(t, mgr.saveMetadata(meta)) + return id +} + +func TestEnsureShareMemoryTemplate_AutoPromoteAndReuse(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := stagedStandbySource(t, mgr, "share-mem-source") + + tpl1, err := mgr.ensureShareMemoryTemplate(ctx, sourceID) + require.NoError(t, err) + require.NotNil(t, tpl1) + assert.Equal(t, sourceID, tpl1.SourceInstanceID) + assert.Equal(t, shareMemoryTemplateName(sourceID), tpl1.Name) + + // Source is now flagged as a template parent. + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + assert.True(t, meta.StoredMetadata.IsTemplate) + assert.Equal(t, tpl1.ID, meta.StoredMetadata.TemplateID) + + // Second call returns the same registry entry — no duplicate promotion. + tpl2, err := mgr.ensureShareMemoryTemplate(ctx, sourceID) + require.NoError(t, err) + assert.Equal(t, tpl1.ID, tpl2.ID) +} + +func TestEnsureShareMemoryTemplate_RejectsNonStandby(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + // Same staged source layout but no snapshot dir → state derives as Stopped. + id := "share-mem-stopped-source" + require.NoError(t, mgr.ensureDirectories(id)) + now := time.Now() + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeFirecracker, + HypervisorVersion: "test", + DataDir: mgr.paths.InstanceDir(id), + VsockCID: 45, + VsockSocket: mgr.paths.InstanceVsockSocket(id), + }})) + + _, err := mgr.ensureShareMemoryTemplate(ctx, id) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) +} + +func TestTemplateGuard_ReturnsInvalidStateNotUnsupported(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + + // Locked: a template parent should return ErrInvalidState (409), not + // ErrNotSupported (501) — the lock is transient (resolves once forks + // are deleted), not a hypervisor capability gap. + stored := &StoredMetadata{Id: "src", IsTemplate: true, TemplateID: "tpl-xyz"} + err := mgr.templateGuard(stored, "start") + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidState) + assert.NotErrorIs(t, err, ErrNotSupported) + + // Not a template: no error. + stored.IsTemplate = false + require.NoError(t, mgr.templateGuard(stored, "start")) +} + +func TestHydrateForkLockState(t *testing.T) { + t.Parallel() + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := stagedStandbySource(t, mgr, "share-mem-hydrate-source") + tpl, err := mgr.ensureShareMemoryTemplate(ctx, sourceID) + require.NoError(t, err) + + // Zero forks initially. + inst, err := mgr.GetInstance(ctx, sourceID) + require.NoError(t, err) + assert.Equal(t, 0, inst.ForkCount) + assert.False(t, inst.MemLocked) + + // Bump refcount and re-read: ForkCount/MemLocked should reflect it. + require.NoError(t, mgr.bumpTemplateForkRefcount(ctx, tpl)) + inst, err = mgr.GetInstance(ctx, sourceID) + require.NoError(t, err) + assert.Equal(t, 1, inst.ForkCount) + assert.True(t, inst.MemLocked) +} diff --git a/lib/instances/start.go b/lib/instances/start.go index 8da3026e..79b855d7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -47,6 +47,10 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if err := m.templateGuard(stored, "start"); err != nil { + log.ErrorContext(ctx, "refusing to start template instance", "instance_id", id, "template_id", stored.TemplateID) + return nil, err + } // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil diff --git a/lib/instances/templates.go b/lib/instances/templates.go new file mode 100644 index 00000000..fd4f8541 --- /dev/null +++ b/lib/instances/templates.go @@ -0,0 +1,393 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/templates" + "github.com/nrednav/cuid2" +) + +// PromoteToTemplateRequest configures a Standby instance promotion into a +// fork-only template parent. +type PromoteToTemplateRequest struct { + // Name is the template's user-facing label. Must be unique. Required. + Name string + // Tags is optional user metadata. + Tags map[string]string +} + +// promoteToTemplate marks a Standby instance as a fork-only template parent +// and registers its metadata in the templates registry. The instance itself +// stays where it is on disk; what changes is the StoredMetadata flag and +// the new entry in the registry. Subsequent forks descend from this +// instance's snapshot directory. +// +// PR 2 ships only the lifecycle plumbing. PR 3 wires the resulting template +// into the firecracker fork path so forks share the template's mem-file +// instead of copying it. +func (m *manager) promoteToTemplate(ctx context.Context, instanceID string, req PromoteToTemplateRequest) (*templates.Template, error) { + log := logger.FromContext(ctx) + if m.templateRegistry == nil { + return nil, fmt.Errorf("%w: template registry not configured", ErrNotSupported) + } + if req.Name == "" { + return nil, fmt.Errorf("%w: template name is required", ErrInvalidRequest) + } + + meta, err := m.loadMetadata(instanceID) + if err != nil { + return nil, err + } + stored := &meta.StoredMetadata + inst := m.toInstance(ctx, meta) + + if inst.State != StateStandby { + return nil, fmt.Errorf("%w: can only promote a Standby instance to a template (got %s)", ErrInvalidState, inst.State) + } + if !inst.HasSnapshot { + return nil, fmt.Errorf("%w: instance %s has no snapshot to promote", ErrInvalidState, instanceID) + } + if stored.IsTemplate { + return nil, fmt.Errorf("%w: instance %s is already a template", ErrAlreadyExists, instanceID) + } + if existing, err := m.templateRegistry.GetByName(ctx, req.Name); err == nil { + return nil, fmt.Errorf("%w: template name %q already registered as id=%s", ErrAlreadyExists, req.Name, existing.ID) + } else if !errors.Is(err, templates.ErrNotFound) { + return nil, fmt.Errorf("check template name: %w", err) + } + + templateID := cuid2.Generate() + + tpl := &templates.Template{ + ID: templateID, + Name: req.Name, + SourceInstanceID: instanceID, + Image: stored.Image, + HypervisorType: stored.HypervisorType, + HypervisorVersion: stored.HypervisorVersion, + MemoryBytes: stored.Size + stored.HotplugSize, + VCPUs: stored.Vcpus, + CreatedAt: m.now().UTC(), + } + for k, v := range req.Tags { + if tpl.Tags == nil { + tpl.Tags = map[string]string{} + } + tpl.Tags[k] = v + } + + if err := m.templateRegistry.Save(ctx, tpl); err != nil { + return nil, fmt.Errorf("save template: %w", err) + } + + stored.IsTemplate = true + stored.TemplateID = templateID + if err := m.saveMetadata(meta); err != nil { + // Best-effort rollback of the registry entry. If this fails the + // operator can manually delete the orphan via DeleteTemplate. + if delErr := m.templateRegistry.Delete(ctx, templateID); delErr != nil { + log.WarnContext(ctx, "failed to roll back template registry entry after metadata save failure", + "template_id", templateID, "error", delErr) + } + return nil, fmt.Errorf("persist template flag on instance: %w", err) + } + + log.InfoContext(ctx, "promoted instance to template", + "instance_id", instanceID, "template_id", templateID, "name", req.Name) + return tpl, nil +} + +// listTemplates returns all templates, optionally filtered. +func (m *manager) listTemplates(ctx context.Context, filter *templates.ListFilter) ([]*templates.Template, error) { + if m.templateRegistry == nil { + return nil, nil + } + return m.templateRegistry.List(ctx, filter) +} + +// getTemplate looks up a template by ID. +func (m *manager) getTemplate(ctx context.Context, templateID string) (*templates.Template, error) { + if m.templateRegistry == nil { + return nil, fmt.Errorf("%w: template registry not configured", ErrNotSupported) + } + return m.templateRegistry.Get(ctx, templateID) +} + +// deleteTemplate removes a template from the registry. The underlying +// source instance is not deleted; callers can decide whether to delete it +// separately. Refuses when ForkCount > 0. +func (m *manager) deleteTemplate(ctx context.Context, templateID string) error { + if m.templateRegistry == nil { + return fmt.Errorf("%w: template registry not configured", ErrNotSupported) + } + tpl, err := m.templateRegistry.Get(ctx, templateID) + if err != nil { + return err + } + + if err := m.templateRegistry.Delete(ctx, templateID); err != nil { + return err + } + + // Best-effort: clear the IsTemplate flag on the source instance if it + // still exists, so the operator can resume/delete it normally. + if tpl != nil && tpl.SourceInstanceID != "" { + meta, err := m.loadMetadata(tpl.SourceInstanceID) + if err == nil { + meta.StoredMetadata.IsTemplate = false + meta.StoredMetadata.TemplateID = "" + _ = m.saveMetadata(meta) + } + } + return nil +} + +// touchTemplateUsage updates LastUsedAt on a template. Cheap; called +// whenever a fork is created from the template. +func (m *manager) touchTemplateUsage(ctx context.Context, templateID string) { + if m.templateRegistry == nil || templateID == "" { + return + } + tpl, err := m.templateRegistry.Get(ctx, templateID) + if err != nil { + return + } + tpl.LastUsedAt = m.now().UTC() + _ = m.templateRegistry.Save(ctx, tpl) +} + +// templateGuard returns an error when the instance is a template parent. +// Templates must not be Started or Restored — the snapshot is shared with +// live forks and resuming it would corrupt them. +// +// Returns ErrInvalidState (409) so callers see this as a transient +// state-conflict (resolves once forks are deleted), not as a hypervisor +// capability gap (501). +func (m *manager) templateGuard(stored *StoredMetadata, op string) error { + if stored == nil || !stored.IsTemplate { + return nil + } + return fmt.Errorf("%w: cannot %s instance %s while it is mem-locked by live forks; delete the forks (or wait for them to exit) first", ErrInvalidState, op, stored.Id) +} + +// validateForkResolvedFromTemplate confirms a fork-from-template request +// targets a hypervisor compatible with the template. The actual fork +// mechanics live in PR 3. +func validateForkResolvedFromTemplate(tpl *templates.Template, hvType hypervisor.Type) error { + if tpl == nil { + return fmt.Errorf("%w: nil template", ErrInvalidRequest) + } + if hvType != "" && tpl.HypervisorType != hvType { + return fmt.Errorf( + "%w: template hypervisor %s does not match requested %s", + ErrInvalidRequest, tpl.HypervisorType, hvType, + ) + } + return nil +} + +// templateForFork resolves a template by id-or-name. Empty input returns +// (nil, nil) so callers can treat "no template" as the ordinary fork path. +func (m *manager) templateForFork(ctx context.Context, idOrName string) (*templates.Template, error) { + if idOrName == "" || m.templateRegistry == nil { + return nil, nil + } + tpl, err := m.templateRegistry.Get(ctx, idOrName) + if err == nil { + return tpl, nil + } + if !errors.Is(err, templates.ErrNotFound) { + return nil, err + } + return m.templateRegistry.GetByName(ctx, idOrName) +} + +// templateRegistryRef exposes the registry to siblings within the package +// (e.g. fork.go for refcount bumps in PR 3/4). External packages must use +// the manager interface methods. +func (m *manager) templateRegistryRef() templates.Registry { + return m.templateRegistry +} + +// nowOrDefault returns the configured clock or time.Now if unset. Useful +// in code paths that may be called before NewManager has stamped a clock. +func (m *manager) nowOrDefault() time.Time { + if m.now == nil { + return time.Now() + } + return m.now() +} + +// resolveForkFromTemplateRequest expands a ForkInstanceRequest with a +// non-empty TemplateID into (sourceInstanceID, *Template). Returns +// (instanceID, nil, nil) when TemplateID is empty so callers fall through +// to the ordinary fork path. Returns an error when the caller passed both +// instanceID and TemplateID, when the registry is unconfigured, or when +// the template cannot be resolved. +func (m *manager) resolveForkFromTemplateRequest(ctx context.Context, instanceID string, req ForkInstanceRequest) (string, *templates.Template, error) { + if req.TemplateID == "" { + return instanceID, nil, nil + } + if instanceID != "" { + return "", nil, fmt.Errorf("%w: pass either an instance id or a template id, not both", ErrInvalidRequest) + } + if m.templateRegistry == nil { + return "", nil, fmt.Errorf("%w: template registry not configured", ErrNotSupported) + } + tpl, err := m.templateForFork(ctx, req.TemplateID) + if err != nil { + return "", nil, fmt.Errorf("resolve template %q: %w", req.TemplateID, err) + } + if tpl == nil { + return "", nil, fmt.Errorf("%w: template %q not found", ErrNotFound, req.TemplateID) + } + if tpl.SourceInstanceID == "" { + return "", nil, fmt.Errorf("%w: template %s has no source instance", ErrInvalidState, tpl.ID) + } + return tpl.SourceInstanceID, tpl, nil +} + +// installForkSharedMemFile arranges the fork's snapshot directory so the +// guest mem-file is a hardlink to the template's snapshot mem-file +// instead of a per-fork copy. firecracker mmaps the mem-file MAP_PRIVATE +// during restore, so all forks COW from the same backing inode. +// +// Layout: dst is the fork's data dir. The snapshot dir is at +// /snapshots/snapshot-latest, and the mem-file lives at +// /memory. The hardlink shares the inode with the +// template's source instance's standby snapshot mem-file. +// +// We use a hardlink rather than a symlink because RestoreVM temporarily +// aliases the source data dir to the fork data dir while firecracker +// loads the snapshot (see withSnapshotSourceDirAlias). A symlink whose +// target traverses the source dir would resolve back into the fork dir +// during that window and trip ELOOP; a hardlink resolves by inode so +// the alias has no effect on it. Hardlinks require both paths on the +// same filesystem, which holds for our standard data-dir layout. +func (m *manager) installForkSharedMemFile(forkDataDir string, tpl *templates.Template) error { + if tpl == nil { + return nil + } + srcMem := filepath.Join(m.paths.InstanceSnapshotLatest(tpl.SourceInstanceID), templateSharedMemFileName) + if _, err := os.Stat(srcMem); err != nil { + return fmt.Errorf("stat template mem-file: %w", err) + } + dstSnapshotDir := filepath.Join(forkDataDir, "snapshots", "snapshot-latest") + if err := os.MkdirAll(dstSnapshotDir, 0o755); err != nil { + return fmt.Errorf("ensure fork snapshot dir: %w", err) + } + dstMem := filepath.Join(dstSnapshotDir, templateSharedMemFileName) + // Tolerate a leftover entry (e.g. from a partial copy that wasn't fully + // skipped on a different filesystem layout). + _ = os.Remove(dstMem) + if err := os.Link(srcMem, dstMem); err != nil { + return fmt.Errorf("hardlink shared mem-file: %w", err) + } + return nil +} + +// templateSharedMemFileRelPath is the relative path under the source data +// dir that points at the snapshotted guest mem-file. Encoded here so the +// fork copy step can skip it without importing firecracker internals. +const ( + templateSharedMemFileName = "memory" + templateSharedMemFileRelPath = "snapshots/snapshot-latest/memory" +) + +// bumpTemplateForkRefcount records that a fork now depends on a template. +// Best-effort touch of LastUsedAt happens alongside. +func (m *manager) bumpTemplateForkRefcount(ctx context.Context, tpl *templates.Template) error { + if tpl == nil || m.templateRegistry == nil { + return nil + } + if _, err := m.templateRegistry.IncrementForkCount(ctx, tpl.ID); err != nil { + return fmt.Errorf("increment template fork count: %w", err) + } + m.touchTemplateUsage(ctx, tpl.ID) + return nil +} + +// dropTemplateForkRefcount mirrors bumpTemplateForkRefcount and is called +// when a fork instance is deleted. Missing templates are tolerated so +// orphaned forks don't block delete. +func (m *manager) dropTemplateForkRefcount(ctx context.Context, templateID string) { + if templateID == "" || m.templateRegistry == nil { + return + } + if _, err := m.templateRegistry.DecrementForkCount(ctx, templateID); err != nil { + log := logger.FromContext(ctx) + log.WarnContext(ctx, "failed to decrement template fork refcount", + "template_id", templateID, "error", err) + } +} + +// ensureShareMemoryTemplate resolves (or creates) the template entry that +// backs ShareMemory=true forks against the given source instance. If the +// source is already a template parent, the existing entry is returned. +// Otherwise the source is auto-promoted with a deterministic, internal +// name derived from its instance ID — the public API never exposes the +// templates resource, so the name is purely a registry detail. +// +// The source must be in Standby; this is checked here so callers see a +// clear error before the fork machinery starts allocating fork state. +func (m *manager) ensureShareMemoryTemplate(ctx context.Context, instanceID string) (*templates.Template, error) { + if instanceID == "" { + return nil, fmt.Errorf("%w: share_memory requires a source instance id", ErrInvalidRequest) + } + if m.templateRegistry == nil { + return nil, fmt.Errorf("%w: template registry not configured", ErrNotSupported) + } + meta, err := m.loadMetadata(instanceID) + if err != nil { + return nil, err + } + stored := &meta.StoredMetadata + if stored.IsTemplate && stored.TemplateID != "" { + tpl, err := m.templateRegistry.Get(ctx, stored.TemplateID) + if err != nil { + return nil, fmt.Errorf("load existing share-memory template: %w", err) + } + return tpl, nil + } + inst := m.toInstance(ctx, meta) + if inst.State != StateStandby { + return nil, fmt.Errorf("%w: share_memory requires the source to be in Standby (got %s)", ErrInvalidState, inst.State) + } + return m.promoteToTemplate(ctx, instanceID, PromoteToTemplateRequest{ + Name: shareMemoryTemplateName(instanceID), + }) +} + +// shareMemoryTemplateName computes the registry name used for auto-promoted +// share-memory templates. Encoded as a function so tests can assert that +// repeated ShareMemory forks against the same source resolve the same +// registry entry. +func shareMemoryTemplateName(instanceID string) string { + return "share-mem-" + instanceID +} + +// hydrateForkLockState fills in ForkCount/MemLocked on inst by looking up +// the instance's template entry. Non-fatal: any registry lookup error +// leaves the fields at their zero value so callers see "not locked" rather +// than a hard failure. +func hydrateForkLockState(ctx context.Context, registry templates.Registry, inst *Instance) { + if inst == nil || registry == nil { + return + } + if !inst.IsTemplate || inst.TemplateID == "" { + return + } + tpl, err := registry.Get(ctx, inst.TemplateID) + if err != nil || tpl == nil { + return + } + inst.ForkCount = tpl.ForkCount + inst.MemLocked = tpl.ForkCount > 0 +} diff --git a/lib/instances/types.go b/lib/instances/types.go index 56243a0d..4d9854ae 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -153,6 +153,15 @@ type StoredMetadata struct { // Exit information (populated from serial console sentinel when VM stops) ExitCode *int // App exit code, nil if VM hasn't exited ExitMessage string // Human-readable description of exit (e.g., "command not found", "killed by signal 9 (SIGKILL) - OOM") + + // Template-related fields. These are zero-valued for ordinary instances; + // the templates package owns the lifecycle and refcount. Forks and + // templates persist these fields so the manager can refuse to Start a + // template directly and so a deleted fork can decrement its template's + // refcount. + IsTemplate bool // true once an instance has been promoted to a template parent + TemplateID string // when set, this instance is the canonical source for the named template + ForkOfTemplate string // when set, this instance was forked from the named template } // Instance represents a virtual machine instance with derived runtime state @@ -164,6 +173,15 @@ type Instance struct { StateError *string // Error message if state couldn't be determined (non-nil when State=Unknown) HasSnapshot bool // Derived from filesystem check BootMarkersHydrated bool // True when missing boot markers were hydrated from logs in this read + + // ForkCount is the number of live forks created against this instance + // with ShareMemory=true. Derived from the templates registry when the + // instance is a fan-out parent; zero otherwise. + ForkCount int + // MemLocked is true iff ForkCount > 0. While true, start/restore/delete + // of this instance fails with ErrInvalidState because the snapshot + // mem-file is being served to live forks. + MemLocked bool } // GetHypervisorType returns the hypervisor type as a string. @@ -248,6 +266,23 @@ type ForkInstanceRequest struct { Name string // Required: name for the new forked instance FromRunning bool // Optional: allow forking from Running by auto standby/fork/restore TargetState State // Optional: desired final state of forked instance (Stopped, Standby, Running). Empty means inherit source state. + + // TemplateID resolves the source instance from the template registry by + // id-or-name. When set, the source instance id passed to ForkInstance is + // ignored (must be empty). The fork's mem-file is hardlinked to the + // template's mem-file instead of being copied per-fork, so many forks + // fan out from the same warm guest memory. + TemplateID string + + // ShareMemory opts the fork into mem-file sharing with the source + // instance: instead of copying the snapshot mem-file, the fork's + // hypervisor reads pages from the source's mem-file (via uffd or + // hardlink, depending on hypervisor). Requires the source to be in + // Standby. The first such fork against a source auto-promotes that + // source so subsequent ShareMemory forks reuse the same registry entry; + // while any are alive, the source is mem-locked (start/restore/delete + // return ErrInvalidState). + ShareMemory bool } // SnapshotKind determines how snapshot data is captured and restored. diff --git a/lib/instances/uffd.go b/lib/instances/uffd.go new file mode 100644 index 00000000..c9b8c9d0 --- /dev/null +++ b/lib/instances/uffd.go @@ -0,0 +1,197 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/templates" + "github.com/kernel/hypeman/lib/uffd" +) + +// uffdTracker owns one uffd.Server per template mem-file and tracks which +// forks are currently restored against each one. The first +// acquireUffdForFork for a template lazily starts the server; the last +// releaseUffdForFork tears it down. This keeps the server out of the +// critical path for non-template forks (the symlink-only path from +// PR 3) and avoids leaking page-server goroutines once a template +// becomes idle. +// +// Lifecycle assumption: this PR scopes uffd to the *active* fork-create +// path. After a hypeman restart, any firecracker process previously +// backed by a uffd server has no one to handle its faults until those +// forks are themselves restarted; that gap is documented in +// recoverTemplateForkRefcounts and is the next follow-up. +type uffdTracker struct { + mu sync.Mutex + entries map[string]*uffdEntry +} + +type uffdEntry struct { + server *uffd.Server + socketDir string + forks map[string]struct{} +} + +func newUffdTracker() *uffdTracker { + return &uffdTracker{entries: map[string]*uffdEntry{}} +} + +// acquireUffdForFork ensures a uffd.Server is running for the template, +// registers forkID with it, and returns the per-fork UDS path. Callers +// must call releaseUffdForFork(templateID, forkID) once the fork no +// longer exists, even if firecracker never connected. +func (t *uffdTracker) acquireUffdForFork(ctx context.Context, tpl *templates.Template, memFilePath, socketDir, forkID string) (string, error) { + if tpl == nil { + return "", errors.New("uffd: template is required") + } + if forkID == "" { + return "", errors.New("uffd: fork id is required") + } + t.mu.Lock() + entry, ok := t.entries[tpl.ID] + if !ok { + srv, err := uffd.NewServer(uffd.Config{ + MemFilePath: memFilePath, + SocketDir: socketDir, + }) + if err != nil { + t.mu.Unlock() + return "", fmt.Errorf("uffd: start server for template %s: %w", tpl.ID, err) + } + entry = &uffdEntry{server: srv, socketDir: socketDir, forks: map[string]struct{}{}} + t.entries[tpl.ID] = entry + } + t.mu.Unlock() + + socketPath, err := entry.server.RegisterFork(ctx, forkID) + if err != nil { + t.maybeCloseEmpty(tpl.ID) + return "", fmt.Errorf("uffd: register fork %s with template %s: %w", forkID, tpl.ID, err) + } + + t.mu.Lock() + entry.forks[forkID] = struct{}{} + t.mu.Unlock() + return socketPath, nil +} + +// releaseUffdForFork unregisters the fork from its template's server +// and shuts the server down once no forks remain. It is safe to call +// for templates that never had a server (returns nil). +func (t *uffdTracker) releaseUffdForFork(templateID, forkID string) error { + if templateID == "" || forkID == "" { + return nil + } + t.mu.Lock() + entry, ok := t.entries[templateID] + if !ok { + t.mu.Unlock() + return nil + } + if _, present := entry.forks[forkID]; !present { + t.mu.Unlock() + return nil + } + delete(entry.forks, forkID) + srv := entry.server + socketDir := entry.socketDir + empty := len(entry.forks) == 0 + if empty { + delete(t.entries, templateID) + } + t.mu.Unlock() + + var firstErr error + if err := srv.UnregisterFork(forkID); err != nil { + firstErr = err + } + if empty { + if err := srv.Close(); err != nil && firstErr == nil { + firstErr = err + } + if socketDir != "" { + _ = os.RemoveAll(socketDir) + } + } + return firstErr +} + +// maybeCloseEmpty drops a template's entry when no forks ever attached +// to a freshly-created server (acquire-then-RegisterFork failure). +func (t *uffdTracker) maybeCloseEmpty(templateID string) { + t.mu.Lock() + entry, ok := t.entries[templateID] + if !ok || len(entry.forks) > 0 { + t.mu.Unlock() + return + } + delete(t.entries, templateID) + srv := entry.server + socketDir := entry.socketDir + t.mu.Unlock() + _ = srv.Close() + if socketDir != "" { + _ = os.RemoveAll(socketDir) + } +} + +// closeAll tears down every server. Called by the manager during +// shutdown so the uffd goroutines and mem-file fds don't leak. +func (t *uffdTracker) closeAll() error { + t.mu.Lock() + entries := t.entries + t.entries = map[string]*uffdEntry{} + t.mu.Unlock() + + var firstErr error + for _, entry := range entries { + for forkID := range entry.forks { + _ = entry.server.UnregisterFork(forkID) + } + if err := entry.server.Close(); err != nil && firstErr == nil { + firstErr = err + } + if entry.socketDir != "" { + _ = os.RemoveAll(entry.socketDir) + } + } + return firstErr +} + +// hasFork is a test-only helper that reports whether forkID is currently +// tracked under templateID. +func (t *uffdTracker) hasFork(templateID, forkID string) bool { + t.mu.Lock() + defer t.mu.Unlock() + entry, ok := t.entries[templateID] + if !ok { + return false + } + _, present := entry.forks[forkID] + return present +} + +// uffdSupportedForFork reports whether the manager will try to serve a +// fork's mem-file via uffd. Only firecracker is wired to consume the +// backend; other hypervisors fall back to the symlink-only path. +func uffdSupportedForFork(hvType hypervisor.Type) bool { + return hvType == hypervisor.TypeFirecracker +} + +// acquireForkUffdIfApplicable returns a per-fork uffd UDS path when the +// fork should be backed by a userfaultfd page server, or "" when it +// should fall back to the symlink-only path. Failure to start the +// server is bubbled up as an error so the caller can abort the fork. +func (m *manager) acquireForkUffdIfApplicable(ctx context.Context, tpl *templates.Template, forkID string, hvType hypervisor.Type) (string, error) { + if tpl == nil || !uffdSupportedForFork(hvType) || m.uffd == nil { + return "", nil + } + memFilePath := filepath.Join(m.paths.InstanceSnapshotLatest(tpl.SourceInstanceID), templateSharedMemFileName) + socketDir := m.paths.TemplateUffdDir(tpl.ID) + return m.uffd.acquireUffdForFork(ctx, tpl, memFilePath, socketDir, forkID) +} diff --git a/lib/instances/uffd_test.go b/lib/instances/uffd_test.go new file mode 100644 index 00000000..604b946c --- /dev/null +++ b/lib/instances/uffd_test.go @@ -0,0 +1,93 @@ +//go:build linux + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kernel/hypeman/lib/templates" +) + +func writeUffdTrackerMemFile(t *testing.T, dir string) string { + t.Helper() + p := filepath.Join(dir, "memory") + require.NoError(t, os.WriteFile(p, make([]byte, 4096), 0o644)) + return p +} + +func TestUffdTracker_AcquireAndReleaseLifecycle(t *testing.T) { + tracker := newUffdTracker() + t.Cleanup(func() { _ = tracker.closeAll() }) + + tplDir := t.TempDir() + memPath := writeUffdTrackerMemFile(t, tplDir) + tpl := &templates.Template{ID: "tpl-1", SourceInstanceID: "src-1"} + + socketDir := filepath.Join(tplDir, "uffd") + socketA, err := tracker.acquireUffdForFork(context.Background(), tpl, memPath, socketDir, "fork-a") + require.NoError(t, err) + require.NotEmpty(t, socketA) + require.True(t, tracker.hasFork(tpl.ID, "fork-a")) + + // Second fork against the same template reuses the existing server and + // returns a distinct UDS path. + socketB, err := tracker.acquireUffdForFork(context.Background(), tpl, memPath, socketDir, "fork-b") + require.NoError(t, err) + require.NotEmpty(t, socketB) + require.NotEqual(t, socketA, socketB) + require.True(t, tracker.hasFork(tpl.ID, "fork-b")) + + // Releasing one fork keeps the server alive for the remaining fork. + require.NoError(t, tracker.releaseUffdForFork(tpl.ID, "fork-a")) + require.False(t, tracker.hasFork(tpl.ID, "fork-a")) + require.True(t, tracker.hasFork(tpl.ID, "fork-b")) + + // Releasing the last fork tears the server down. + require.NoError(t, tracker.releaseUffdForFork(tpl.ID, "fork-b")) + require.False(t, tracker.hasFork(tpl.ID, "fork-b")) + + // A subsequent acquire should be able to start a fresh server. + socketC, err := tracker.acquireUffdForFork(context.Background(), tpl, memPath, socketDir, "fork-c") + require.NoError(t, err) + require.NotEmpty(t, socketC) + require.True(t, tracker.hasFork(tpl.ID, "fork-c")) +} + +func TestUffdTracker_ReleaseUnknownFork_NoError(t *testing.T) { + tracker := newUffdTracker() + require.NoError(t, tracker.releaseUffdForFork("missing-template", "missing-fork")) +} + +func TestUffdTracker_AcquireRejectsEmpty(t *testing.T) { + tracker := newUffdTracker() + _, err := tracker.acquireUffdForFork(context.Background(), nil, "/tmp/x", "/tmp/y", "fork") + assert.Error(t, err) + + tplDir := t.TempDir() + memPath := writeUffdTrackerMemFile(t, tplDir) + tpl := &templates.Template{ID: "tpl-1", SourceInstanceID: "src-1"} + _, err = tracker.acquireUffdForFork(context.Background(), tpl, memPath, filepath.Join(tplDir, "uffd"), "") + assert.Error(t, err) +} + +func TestUffdTracker_CloseAll(t *testing.T) { + tracker := newUffdTracker() + tplDir := t.TempDir() + memPath := writeUffdTrackerMemFile(t, tplDir) + tpl := &templates.Template{ID: "tpl-1", SourceInstanceID: "src-1"} + + _, err := tracker.acquireUffdForFork(context.Background(), tpl, memPath, filepath.Join(tplDir, "uffd"), "fork-a") + require.NoError(t, err) + _, err = tracker.acquireUffdForFork(context.Background(), tpl, memPath, filepath.Join(tplDir, "uffd"), "fork-b") + require.NoError(t, err) + + require.NoError(t, tracker.closeAll()) + assert.False(t, tracker.hasFork(tpl.ID, "fork-a")) + assert.False(t, tracker.hasFork(tpl.ID, "fork-b")) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 093dd8f5..fe95f3a5 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -683,6 +683,14 @@ type ForkInstanceRequest struct { // Name Name for the forked instance (lowercase letters, digits, and dashes only; cannot start or end with a dash) Name string `json:"name"` + // ShareMemory Share the source instance's snapshot mem-file with the fork instead of copying it. + // Requires the source to be in Standby state. While at least one fork created with + // share_memory=true is alive, the source is mem-locked: start/restore/delete on the + // source return 409 until all such forks are deleted (see Instance.fork_count and + // Instance.mem_locked). Forks created with share_memory=false are unaffected and may + // be created against the same locked source. + ShareMemory *bool `json:"share_memory,omitempty"` + // TargetState Target state for the forked instance after fork completes TargetState *ForkTargetState `json:"target_state,omitempty"` } @@ -875,6 +883,10 @@ type Instance struct { // ExitMessage Human-readable description of exit (e.g., "command not found", "killed by signal 9 (SIGKILL) - OOM") ExitMessage *string `json:"exit_message,omitempty"` + // ForkCount Number of live forks that share this instance's snapshot mem-file via share_memory=true. + // Zero for instances that have never been used as a fan-out parent. Read-only. + ForkCount *int `json:"fork_count,omitempty"` + // Gpu GPU information attached to the instance Gpu *InstanceGPU `json:"gpu,omitempty"` @@ -893,6 +905,10 @@ type Instance struct { // Image OCI image reference Image string `json:"image"` + // MemLocked True iff fork_count > 0 — i.e. at least one share_memory fork is alive against this + // instance. While true, start/restore/delete on this instance return 409. Read-only. + MemLocked *bool `json:"mem_locked,omitempty"` + // Name Human-readable name Name string `json:"name"` @@ -15654,273 +15670,279 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3YbubE3+ipYfbJXpISkqKttZc3aRyPZHu2xbB3Lds7O0B8FdoMkRt1AD4CmRHv5", - "3zxAHjFP8i0UgL4RTbZkXazYe2clMrsbl0KhUFWo+tXnIORJyhlhSgb7nwMZTkmC4c8DpXA4/cDjLCFv", - "yR8ZkUr/nAqeEqEogZcSnjE1TLGa6n9FRIaCpopyFuwHp1hN0eWUCIJm0AqSU57FERoRBN+RKOgE5Aon", - "aUyC/WAjYWojwgoHnUDNU/2TVIKySfClEwiCI87iuelmjLNYBftjHEvSqXV7optGWCL9SRe+ydsbcR4T", - "zIIv0OIfGRUkCvZ/K0/jY/4yH/1OQqU7P8gUP1OYRaP5KY9pOF+c7CvKsivoDeFM8QQrGiJpvkEpfIRG", - "WJIIcYZwqOiMIMpGPGMRend4ikLOGAl1Y3LA+EgSMSMRGgueIDUlaMqlgneUwOEFUngUk96ABZ3aehCm", - "n0SrqfT3KVFTIjyDpRLZVtCYC6SmVCLK9NOQ9MoLpkRGFinbCWgUk6GiCeGZWiTUL/wSxZxNYFquXZRk", - "UqEpnhH0iQiO/shwTMdzyibNRBqRMRcE/TJPSYIZSmMcEomoQpQp7mZjaFTw2G7iYy46YVyQYUSkogzr", - "9ocpF2ZHVEf/Bv7AMSq9C0OD95GaYuW4nHGFLghJqxPFl/iiSsbftrY6z/r9/sdOQBVJzLbCVzTJkmB/", - "b3d3e7cTJJSZf2/mo6dMkQkRevj2FywEnpemI3kmQjIMaSSWzSSMKWEKHR4fvb3hBILNfg/+f+Np0Ak2", - "n231Nveewr8394LytBYIXx35l+Vb70xhlclFGWR209AyyrDEJIuzfp0lIyIQH6MwE4IwFc8RbCkStWC6", - "yrT7vqUIORvTSSbcFvRtuQo5p1gizIzQ6NbkRdFYq30XaiEW8Us2FCTBlGkaLwzirXuE9A5FdhPpIYWc", - "KcHjWAsFpUiSKul2UUeLcYZwmsY0BNFT2VQ7SV8GnYBlcawf1kZYrDaJ6YTCC61IQ2Vpkdy3SHFEmCIi", - "3+FtSFMRi00dF+T2rkYhF9tLQUlZ6J8uq9M80RJekNBMNz8BKhQZkZAnBOmmqyuw1d/a6/Z3uv29d5tP", - "9vs7+/3dfwSdYMxFglWwH0RYka5e8DbLtFx+HxZU0i8i+2JxVHlo16vJ4HbsEmOp8l0Nm5yq+RB7xvSO", - "JkQqnKR6Y+sxlIjZtK1dg/V1cJRfSuDNryIwI1dqaCnknY+PP8hVSkJ9xHC3PfMTW7fXQXSMMMplgGZX", - "IxiXTuTZV01EECz1gLXeoU+n34KMySzVZyGJhmmMlW5XKynABsOESqk/zX+IqDQbsxM4Jh8yroYiY8y8", - "yIi65OKi/KZtZUjToBNMsRzOJmkWdJadA1Wmhi5IjFMJ7dkVF0MiBBeB0TXnwzEXbpH0IVaQcElTCxSS", - "+ZnloVDQCSoEyOWjm4sbd76q3sFBL8BLwqjpRq+GySwOvNzW4nDzoS2XlEYsG63ULTOyH8uqBIgonjAu", - "FQ1lK7kJp7Fe3oRHHtF5lDeHaESYomNKhFVUCRIZg2PNNYJ0I4gylMnaPsh16SGZaeNnONsZqjBdJErN", - "UigvXumwL46Y0jGXL3++U1YwaXXuXktkhinsySMyo+ZoqSpDdmmGkaAzIjziOz9RjSg076E1vde1CGGc", - "kfUKpdiMRhS3EQcRjGlIPdxzeniMzGN0fITWpuSq2snWk9HToLlJhhMPL/ySJZh19YbQw3Ltw7vltl/t", - "eHV+niTZcCJ4li62fPzm5OQ9goeIgcpYbvHplk/1S0M6xFEkiJT++buH5bH1+/3+Pt7a7/d7fd8oZ4RF", - "XDSS1Dz2k3SzH5ElTbYiqW1/gaSvPxwfHR+gQy5SLsAIWrlxyuQpz6vMNtVV8fH/zxmNo0WuH+mfiRjm", - "h4iPYMdOjTo+cnqC/Q59OEFrWoZEZJRNJpRN1tvwe8g1OfRR5zvEYajIvqPNROW0lBuft6EgeEV3+o1W", - "nS1utcys5DCRTa27V7RETWgcU0lCziJZ7oMytbfTPJnShjEn1EJXz/XPKCFS4glBa+BSAfPDCFOt2Iwx", - "jUm03k6ZbZrM73xUOkIq7A1s0cWjcHNr2ys7Ejwhw4hOrE+sfkTp3zWL6XYUgrf9E4HDvN08oEtBxov9", - "vQDRDZ0IMiaCaB7/yu5SwWeEYWu9/An6Df6fjcJZuGE9hRtAzNPi9S+d4I+MZGSYcknNCBckl32i2QhI", - "jeAL/5jh0bK1LnGUVFgs3x/wxi3sxEKvW0kb67bQqg2erPzknX6nLjtBNOa6REkKNIrI51qp8WgHnCn7", - "oOa+5BMUU2YsDq3ambUAvWqekp9iDiLxluiQk39x8+tx30B4mR8aWtPPOrkCHvNJmZpTgoUakQoxG44w", - "21Axukbyn1a2T+2swpIMl0uQU8oYicBfbDe2eVOrsV4zA3bRBVXDGRHSu+dgWL9ShewbjU3FPLwY05gM", - "p1hOrYMtiqhxFp5WZuLR1iqOeAz2uGsQtAiwX89+Odja3UO2Aw8NredSv7A4k9LXunnzLlJYjHAce3mj", - "md2uf0YvcoifAwpnZdPZk3OgY0wj6QK7mtZOzuTU/AWyW48Kzj4tBjR7xfrvj55JH4KQMFZC4+2NXwfM", - "PcOTmGuazlHG6B9ZRcHuoeMxOIj1QUEjEnUQhgfgd9D234QwIrScKjxDJSUYrZHepNdBA60XdrUW3MVb", - "3X6/2x8EVTU23uka8z7FShGhB/h/fsPdTwfdf/S7zz4Wfw573Y9//ZOPAdpq5k4rtPNcc3u/g9xgy+p6", - "faCrVPkbS//y8H0Sxyz1sZYT113pw+NFxcHMNeLhBRE9yjdiOhJYzDfYhLKr/RgrIlV15svfvVVawDyW", - "EIFNNJmuSYaa0QNsvBbzSyJCLYFjohlPdrQQpkp2ENZ2MwgvpE/Jv6EQM70XjHLBBSIsQpdUTRGG96rU", - "SuZdnNIuNUMNOkGCr14RNlHTYH9ve4HPNZOv2T+6H//iflr/by+riywmHiZ/yzNF2QTB4/K1nhtDfkWz", - "bEUcdbMY1LyEsmPz2ebiHdTXrbCbyLKVNsZc41JrIZS7yFYMZPF+Vxtbicd0eDMjQtDIHcuHJ0doLaYX", - "xO4XJDKGBlm/vx3CC/Ansb+EPEkwi8xv6z30JqFKH4dZccqbK9va7RoJpxwUlTjm17lOA00RDBwcLz3H", - "l5HGS+3DvN3FU/8XLlU3wQxPCJij9kU0EvyC6IGaOwFKJLogc63lzNFEN9qdUQk3PITN0Awbr0NvwN5N", - "uSTmFfdIgm+fzghKeHhhrn6nHCz5GY4zIjvocqpVDvAJEhzbn5G5GBuwqR6kDHlKIm2EmNdgauicsNk5", - "SnAK2xwLAnscJVgRQXFMP5krfLhlIBHVJ9yAEdgYKMV6z4chFxHcsHFEcDgtUeHPEp0bheUcmj+nTLP1", - "udmYtcvqz8Gb9+9+fvP+9dHwzenz1wfHw1+f/6/+2XwU7P/2OTChGrmm8jPBggj0p88w3y9GvY2ICPaD", - "g0xNuaCfjLfmSyfQNJCav3BKezwlDNNeyJOgE/yl/M+PXz46hcy4sWd6G3gG9sWrDJmz1COSjpw3UCLr", - "YXJ3G5pkWkS9PH2/oU/nFEuppoJnk2l1Y1jV4FpbIqLyYkj5cJT6xkTlBTreeIO04oJiqjdorqhs9vsn", - "P2/IQaD/sev+sd5DR2bXwvC1DOLC6k9yqtknj/o4PH2PcBzz0PpQxk0XvK4rn4AnTIl5yqnPiKsJp+LV", - "RRnV7RZPryGKNkaUbUi9DN3wenQHvrmxKfGczajgLNHm3AwLqs9pWd0rr98cPR8+f/0h2NcHQZSF1it5", - "+ubtu2A/2O73+4GPQTUHrZCBL0/fm1tPs21UGmeToaSfPKrEQT4/lJCEC2NC22/Q2rSqaZh9i2BxBsH2", - "y58Nc22+BL5yi2LviPJWTMO1a72XP/u4ZTpPiZhR6fOz/ZI/cyu/GO5T4W1zS5YzLXBxr2S/hDHPom6p", - "y04wpoKEEF6h//UHSbQiP/tUvZbyfOd3f7VSYFdopjhOKSNLVNNvREW85OIi5jjqbt6yhmgvVD2hMeZB", - "dX3zmzXHEgsRZyPMoksaqekw4pdMD9kjV+0TlL+cC9crPRMc//uf//pwUthZmy9HqZW0m1u7Xylpa7JV", - "N+31oeQTyVL/NN6n/kl8OPn3P//lZvKwkzCKyI2UOrv+z00L9aAZG0to3KENN8P56Z0HrChuDWr4HDne", - "W3kN7BPUfEZEjOclwWvHFGz2QfrVRiUoREki+50WoxdIf7xCDOvW3CH/sm7kb/X9gtYzKM+Yftaywp4L", - "bUaSD2Rz68T+ubU4pIYRXdB0CFrzEE9yn++ykNCzC5paVRy+MMsYx0YQRBko7yPOVW/ATISKXjtYYHJF", - "QpB5UmGFDk6PJbqkcQweIhAqi0eLVuxLoU3wulT6v0XGOmiUKa2tc0WQtZugkwzGAi+PCMoYdvfhNd3Z", - "TnAxvADIckEEI/HQ6MayJWXMR8h+1EgcmOoYSxuiJlSWVul19OvJGVo7mjOc0BD9alo94VEWE3RmogvW", - "q9TrDFgqIExBd6L3M7X98jHimerycVcJQtwQE2gs97HZy9rZy9P39rpfrvcG7C3RhCUssoG+7sSxQaAR", - "Z3/WO5ZE1WbL/deI3hTSIRlO5ZSrYZoHTy+TTmf29cIUb+9M6ASzMM2qS7rVaQwCnVGhMhxrWVtRJ70X", - "/CaI3WM2mBj5svli5V4RNKuqN7NtPS6mZYho94bLehwnRlNq7TgpmfILLhRnZ35uN9gV7R8zN5CljqPC", - "1PyKvs5MIwvBO+bnjpvZDah0nNOk5m66HfIcyJJp3ir43MRgGY1QorVzbc1bPtb2+3kHnf+l8oPe+860", - "0PrFJTLUAHnC9E/l9utOiZXugmuFe5cXB8ubr8eBbIx0QrNNpARm0sSoTXFKeugXEOJIkSTVkoxNEJUo", - "D+1CjF/+DXGj1LhPB0wPTZo4EUuO3Gkk6YRRNlnXar4+mHAUGc/SOFOZ0O/NqCyoWWUd571ZiGo1oyNG", - "HkOGBGVhnEUEnTsPz3lVL1z0/yyahNYhtGDhGJKAZQPGntpIMqW71xNOsAqnmk48UyZwzE69GtRX8zKt", - "ulC1Y8mv2m6w/me5uKgnwsw8Jo6enL3kAbdgyT/Z5Aa0iorfRXlB5rDkzh2JFxySZU+k318oiOTxjNhj", - "t+zLHEGqDzeKU+HGNA5J64PU27+e5OLzzq1aCk2v1uSvmgqeFB+pum6yBcdY7d/FhDsppCdn+utow1gS", - "ID6YHvsI1LHzjrGVCHggENPMEqOIChKqheYpmwwYxJCc2196trVzvcm1jnIriVOQhwBKe3lpUWllndoH", - "zeip8YQqRaJOVTe4ICSVqyel1WvruPZ41wW5FNQJMhdU3FI9I2zMRUgSayR8neH4vNSY14y7XhOLIR2G", - "vqUxu/wMyE4hkYkfMusBbtZK2kY9ezGqWW0mhKDa5TmO43O0Zl9aR4L8DpH4dq0YZwWzvzs8dSyQX3t/", - "OOlojtRS4HyqVDrU/yWHehef1xuz37odXmSWPe2DfbWzs21X1TrdzIBrzVb9a96wiOalcep3482a5gs9", - "Shtn0kaVPyw+KTypF5RFbRv4Vb/b6J3LFSNnady1gy4VpJulE4EhxPY23XM3vjcFajZL8BV5vL4wySJD", - "MJOKJ+V4+7VaiAetBoNUiTXjcTfCCoMrs6W/1Qx3MfA4mZumjC3W5IkZTkaeuCH6CVIBJnSCR3NVvT/Y", - "9Gbzfe0lthuLb1maAviNBUmioeLLQ5jpGLl320QsmnwDxYezMeXL0zts/Esl/84cR9au1U1005BadwLo", - "OOHURJgaIoDS+OGkfHfXG7AuHL/76CjvIG82bxKDbokjc3OyxkVpECaRA43m6wijDyc99C4f7Z8l0gbL", - "jLiMhimWaEQIQxm4nuE07JqzuDyATMKhqeqfW9+JSX5YhytKbp/18pxj8NLkGdQQKjWitfmYzElYKHsn", - "jFnZC9bKa7Us8PstmVCpRC3sG629fXG4vb39rO6/3Nrt9je7m7vvNvv7ff2ff7SPEL/9/A5fWwdV2WKD", - "z8rS5/D98dGWdZZW+1GfdvCzp1dXWD3bo5fy2adkJCa/b+N7yQDxi7KjImoOrWWSiK4Tk5qrfLFypZC0", - "hli4G4e43VHEWhGAu+xdQ4l3+s27SG3xBU3bkN3rJ5/UBebKsOvS5BYt+XkKdmexS0oanI1uDKk3jvOI", - "youfBcEXkLK3eG4neELk0Jxn/niGTJogG3JlvRuCczWW5t606vXc3Hmy83R7b+dpv+/J6FhkeB7SYahP", - "oFYDeHN4jGI8JwLBN2gNLrwiNIr5qMrou9t7T5/0n21utR2HueJpR4fc8HJfoTVLkb86nBL3pDKora0n", - "e9vb2/29va2dVqOy/uJWg3K+5YpK8mT7yc7m062dVlTwKfTPXYZNXYH3ZVYemOx+/a+uTElIxzREkKOD", - "9AdoLYEjjOS3VdU9OcKRyz/1nx0K01gujZgwndk3jaMtyWJF05iYZ7AgrXzRMPMjaMmLkMFYnu97vZZs", - "XtLKCAE3l/wVVMkvq5DuxCQ0l5QnSuJo3+zQlXIOVrMY2McmPrBzaMkNr7Tp1I3JjMRlJjBHl8msFQTl", - "fGIWrTIrymY4ptGQsjTzskQjKV9kAnRR0yjCI54pc81oE7SLTiDqGWyPsRbX7ezcF1xcrIwf1Sdxnoe+", - "0it0AI70sXXVwCmOkf3apSiUlL78OtBcmtrnEr01XxgPUfFzmlVRbTrQk/UkMSSIVBwkqXUY2mbaapd+", - "vQWcpS78w/RXyM57in3pjk24wO1a2GJCAH9BrdRYNKe8g/fP4PXW4ej6w5WOlBZ0Z+TyPogO8fpdzbZd", - "yXB6NxRfFoyW+xqKl+AUFjQiPQS7C6JiXH5gbaedKZ6mJMr9P70Bs/Hc+U/S3KDoDw0d1JRQgbigE1rt", - "uOpgu8uotuuwouOmG7Nj+cNFDRUeQvhG86bHY2WwFi5cyhQp5y/ZRQg6wVmOTGElUZU0b3N0jwWKFKGW", - "C0N8efr+urFpqeBj6sMbglgI+9RaZi5q69VO/6y7+f+ZCEzNb6CiUWbiJxIe1YAk7PvtTp6Xp+9Pm8aU", - "Qzug8ugW5pRHvCwDt3IUsZdK9lbSWjCO/fXBkndS6N7PfLrsWOCEjLLxmIhh4nGuvdDPkXnBhDZRhk5+", - "ruqzWm9uazWfVhYHzOYxDm1mfjvqexxytWl0StT86F+ut8Qcw035fHqphH3HpvT10OscTAO9PH0vURGl", - "5PHUVZe3MV7+dDqXNMSxadGk51JWdrABc7bWkE+LD60r0qMn+zFY3EZAa7NJmsE2PHvbPX7zYSOJyKxT", - "GRNEFk15TPS410vSYuay+org/oqQmDV5OgxjyLYbqESrfAe3JlJpv3qoo7jC8VDG3Bes8U4/RPAQrX14", - "YbKu9Ag6KK0spf69RIUKf+95d4yWSE3dnkGHdZdpZYN7bccqGqZxr5SmV+nUt1V+ITg2IKBVfl4EQOIX", - "1YXmF6tBd0wjvn6PXWB4zajxJW8dnhwZhSHkTGHKiEAJUdhCjpZCXEAdCjpBV59RESYJhNqN/7Y8uqXB", - "BV/Oxmp04h4u4HbciQO3Id/8rQlBiFCCGR0TqWy+eaVnOcVbu3v7BhUjIuOd3b1er3fdHJXnRVJKq6XY", - "MCH8pXSVnpx+3TrcQSpKm7l8Dk4P3v0S7AcbmRQbMQ9xvCFHlO2X/p3/s3gAf5h/jijzprC0AlKh4wUA", - "leqVpj6zzO/7JcxLh+/XCtfOb89AZAPkzXnzjRWeaPvEcNzXJhbfGHqkwL9SJciRckBoC/gR+mm5J9Qp", - "RvCO7TNjisYFMsuiD/RG2DpyKfzAAvRASlgOOBDH5q+Qs5neFT70gYoAd8++6v7ARrkMI+rh5L9ba88E", - "SUBW1er9FmzgNF3Ntn5FMZd/bVFXbG605yR6cKl/kzu2au9vJv/zx/8vT5/8vvnHqw8f/nf28n+OXtP/", - "/RCfvvmqDKrlafEPmtt+a+nscLFUyWlvy0onWIUehWrKpWqgsH2CFDfxmj10CIbf/oB10SuqiMDxPhoE", - "tRDhQYDWyBUOlfkKcYZ0UzbTYV1/fGrcP/rjz862/FJvI7IpDcIuSJ7JJLNRxBNM2fqADZhtC7mJSLjT", - "139FKMSpygTRq6d12HiORgJgva15XnTeQZ9xmn5ZHzCwcMmVEnoGKRYqx/FwPQBT2FGZmAH7OolcYrix", - "kAcsP5fyvHDjo+nlThDwzdcjLv1E8ZovXFRTcZ72fRn0EPWlFzKmUhEIzM45W7NRHo6GnvYrouJp/2l/", - "pYKf89AS9oOdsIj375iyxV4yDAxdG8ENEWotfOlaNpk9gn559+5Uk0H/7xlyDRW0yJfYGHkmBlAaH6GK", - "ZSn6bz3woo3C6rackHGSwWdxi6yh5yY89N2rM6SISFzA/lqoyTmmoZ4fXP9TKTPNihSjg8OT5+u9FgUL", - "gLb5+Jes47t8hvXkDus0a/IF5hyv6dtBx0cQnmt3aKHAQVjNCy5QbARMsa/30XtJqrGusFTmVt+sZDwv", - "PG/mBBgE667FtC4p9tHbXG/E+VAqRRKqzrxiX0Kz9uLFxPwstN5ZgB8Xzi6yog0ifLDKg8T1idssCpZv", - "fw/FYc/buO6ST/N6e7vsDNWd+VmjWPvbRk25fXVn+7pG7nURHqpJmKUE3hzkoT06w12gHCwafFdUDRtv", - "8ZF+bO/snVnz4QRNsWR/VvCwZtxsbj9phdepe217/12++eZjM6R8W7qMzvze1uS2XtA4NuEQkk4YjtEz", - "tHZ2/PLX41ev1lEXvXlzUl+KZV/41qcF2IPbGy9P30O6DJZDd4XUHDWJi8hjckWlkosJr61uYpeDS/xS", - "AYDwZhCv3yIqhLu+XpjGfeA9PGRc4LeHNbEUHeJrIR6stnxHCA+NwtWHjlCVs+bn28VquJPhrCwvUlYq", - "XND2jcEROgH1BKweSC0CSYSOTwuQxcKr5ZqvzcnW6tns93ub/TY+vgSHS/o+OThs33l/y3gy9vFoP4z2", - "yfgrfIyWsY32h+NLPJdo4PTzQWAMgpIlUNq2VodvdX+7iEFxM8iJukKxClTiOiAS7dAhvjYlfxnU8lkV", - "ZLm1kvcVlUhahVC4o90GT9ivhtdxnxMU8iyOtCI10lvXGHYksvanJKrAr4bd/p5dMH7JqlM3XlQtAP7I", - "iJijDycnFZ+7IGMLz9ti4hB00bAOPL3WMmyt0LVXjuaGQA33Ac5QF7ul4+7WoRjKTj8XxGk4tIXzr1A/", - "vRfvlJml0XyyZE41t01EZsMs82lV+pFL3Xj//viowhwY720+7T991n062tzr7kT9zS7e3N7rbu3i/ng7", - "fLLdAJDfPvDm5rE01d3cnCoFhAcXqMmEi/b1fsuDYUaZQnmgnN7Ih1o9RSU92CQGgVfimFEFIJCUTXQz", - "4CSwarLJ8DQ4lZRRBZACAGhDmZ4yeGN0Izb8aR+9hHfhEU4gYckNQhtHVUcEjubGEasFg+s6hX8tH/LZ", - "NINyP/CNnGYKQXkoPW1NBmuuLG/CyJh99JrDN8JFqTJet3vM6+ATWHy9biOt2bgkF78KnVmBuY9e5EIy", - "F7NWrK5JYv80stuGVkPY+HoleM+ueKC5pVi5UlxaJzAUDTqBIxTEry1GstlxeZM0yqzou6EgOAYRWkQK", - "ZYrGFiUBZkKhQBJMBMPiNu1kiwhGoqFRAZruG034iVUT8o+coPhwgtYgH/KvyBqV+l/r+d1keVfubD3b", - "ebb3ZOvZXqush2KAqwX8IQRHLQ5upbQP02zoao80TP3w9L0pcRhyJrPEeAns3EtBpqngodZWKUNFMZOi", - "82e9Z+Vkj4hnprCTHZLNDPtSKl+2tPJMwwXbHzSe0fGY/fEpvNj6XdBk82pPbo28xl1RJ82rCR+XXa0L", - "ZiMZdQ2Koz8eHxhKyMaUlbdEwgzQGVEI+KeLcAiHdB7TZFnOJbZYinsZa2d7e/vpk92tVnxlR1faOEOw", - "XxdHeWJHUNpi8CZae3t2hjZKDGfadIGeADDBrALn32fIAjr3q5VAe5v9bR+XNOhLBdfYtmdJI8k/WCXI", - "TsoSHUKzcgVpYZd7qb293X+ys/t0t902dnX3xNVyCeNAPQx5LA5KeeXXwD3/7uAU6dbFGIdVC2Vza3tn", - "d+/J02uNSl1rVIDhY7A3rjGwp0/2dne2tzbb5V75XPA2q7CyYauyy7PpPEzhWQ0PKRZFb6fptPApnobB", - "3pIwxjQ5CF34TO30MRgbQ2FeKxahzcFgnQQLB1eLb1uZaLXCQUY14AKVKi72VrtDb+bdbBbT5jxYLcYX", - "degYM00umyRgoBxvQLtUkBnlmbyFhrgioWamccy5uNa3TfFIb4nMYmVckFSiDyd/BiGimQtJRdJqrL1l", - "vyWpFDec3LU2cIUn/FzdRKxWq9Fm6ZdNuNOwTTvL4mgr278xYynSoipjq+++D3EcZgBehvP11LOC3AOe", - "Kbipn5sokTjmnKFwitmEABi8gUpkE4TRlMdRL/BflcTRcOy9wsgrzPOifrkbhP7M1d9fe8mLknaGlWr4", - "vLuJkSoWuanXopZ8URO3IcNJ0xMrXoIBMJ9UrPmYTyRYgQriX3p19JkUCxPWgpnBqZslxnispm5t6dPe", - "M8Sa9PYdoebo5GNr0VodQ/GckjgUXMqiMPeHk+owlwUw5vXsV99nVwfbgnVlypkk/jLxtiZ8K4eP70D0", - "RIZ9zZEIPAwBoMtqQFscwwSzDJC+SoxMrlIqDHu0uxyfcqmGeTrKNQcr1RBQnDJBipw1d15OIQFgbkQc", - "vOM9F51ouwm58vLGN/h6gav8TTUNsFmmeinqp1Yn50EfGy8m5CzNASqSiuoZJNdJGStgf6iEVmkpWwmt", - "Ma4qYqkEXbPe5qLKb6PqfppKyr7a6Z+1zeZanrx1itX0mI25BxvyGg5/GxLvYhdSIqD8OGcoIoySyBmP", - "ueff+rYgyD6WBEUZsZQzCqnAluDYbG/AgGTOKUbZpCbr6x22ccObMSwHeYJ+7YttrhylPzT7nciAViZI", - "QCJcBGm3inigcuj3FC82LMgki7FA9YzFJUOW8ySm7KJN63KejHhMQ6Q/qF/njHkc88uhfiR/grmst5qd", - "/mBYxBjWrmfM4GyEqVmQWr/FFH7Ss1yvxbeD62XDfL+hv291g+uNG3pBY2KT+t4zelVi9CoKys5Wvyn1", - "oaHRStLDYkLodSW3ZVnfjne5mgd51QRPOL6JAKrdSlQdkZX5+mYLIWbLEj0WXTFozV0KO5SZKl1LaC+t", - "PCHtotzq4Q9uNBuShNXed57uPtlrCbfzVb7OJVWVv8KzOUuWeDQbVuqkjdvs6e7TZ8+2d3afbV3LQeUi", - "ZRrWpylaprw+teIoNafZbh/+71qDMrEy/iE1xMtUB1QpdHLjAX1ZsnWLNOuGa49GnO+4vJLunqXqAW3n", - "Y1yiLR1UVK5SLa81Mh4TMCqHhm7dYjC18PxWYwhxikOq5h6HCb40kO/5K7V04TbetOpgPSS1bVvEBy25", - "ZDYqAjrXXOfoL8a1XuOFp61Ru2Q2anLjv6n3apz4hQ+ofEXU4oamKCyw6C7I53OJZSWqQ/8dAmZzUaut", - "Hj9k3mhfldrxel6YuoiM9KW8+4tQl5e/tpwlt29FSa5TfNkR2rwFr2VDe05kX5nK1VG5NflgD8CbfTUc", - "lfH0lgIWVsD3ilP3+v22qzK3+J05wa7fXykE9Dof1qHFgB/tGCzJi7Y7FZZo4CbFxWpE6TsACDIxBTeC", - "CLLhCPeCEmR/vhNkoIXlOCPKvXumLfosXgIIzRQRM+xxTLkmkHul6kc1kriDrIsPbSbrtVKFO1O/rmZT", - "eFuGj2ktcJgKMqZXS7jFvGCO62r4uLQUiKqg4RKtJfgK7TxB4RQLWRs7o5OpiudVJ+uOJ3vi64o4E6VV", - "5/bw6sVqug8XbzTscpZb923Zs1Kugx/2nUTDZYnuh/lrzmec4jnolo2G4JPtnX5/e6t/o0z320KjL7XT", - "FBFa+s46cypXj+UW8vjPRcjCS0FNUTNHJqkEwck+RFOlOCQoJmPIA8uhYlfa9AtdLx+8vSS1gf85/7uF", - "cpVLrZ/FijjGGSgerh0LEuCmEbhb2mquR/n54rCXJIvlYiZcyBqrx6/udfvb3f7eu83t/d29/c3Nu0iN", - "z4nUFMLz5NPm5ZN4C4934qfzJ39sTp9MtpJtb7rAHRQ+qNURrNVBsHNIiahjUdYxXCWJKSNdmYe9rQ5A", - "XiILzE3Syv1/Pe+DmcFSZeGsOsmyzoBVQZx6Sbb7SGyyo1/qQqkP//ho+bBvFEdWH4ifwepDAX5qNxiA", - "atn8WlyQjLU8d96XXmx98iyNbVx19viCvmFre1e5geI+fq4IxsoOW3ZiL55qHhNuwgVV02T58ZC/lqMM", - "wGX4J6miaiJNDx1PGCDPln/O7z7KxaH1x0EniD/tVPeM/b19SpXNqs8Z0C51WQ1ocTcAwMbLqQCvFKaF", - "MOEJWBAgxE+b3c1ncEMff9r5qd991kN/L0UKdAy1yuTbdG9Xfu23oWEuKEHvNDfnm8+udY3u6LmMg361", - "51LTQWzz7S2PF7Cf7qxwYdOVBS4eL6xxLavozgoN2dNsWNaSIhLjua9YQ4znaETGgG5dsw/LTIZGZEKZ", - "7CBy5YQOlgijsink6rr35SBAXKBBsJsMgh46sCgVYK0WqM6V5gHPt8QnNLE1piyib3NYylbSLn+ibjxc", - "D7rIfeVRz3p+/ezZu/7mtaGLrndM9r4iqvqrzN12Jm6MpWqyLV5hqXKbFInMWhgdKPPDatX+LbS6TYeA", - "8FbAG9hHsublcJjyslD8nCekgyZcoSIRYqWmB8MXGfPyQ3X8RalmgMhoZIitFQzRbkx5MiNdJr6Oj1Aq", - "eJSFRRRwDIPOwpBIOc6g8nSvrVa/+p71Lh0aEF4/5gKtdmg0eTBWp9qSq+b1fk2uVKlLzbDNS73ZX73U", - "d+IF6QRZGq2WYealdhLsWmgkK+JKPT6ZKtlrmmBpMh9bSPS3ZQouGrlQtAqFWiXKUldFUfPUIidJT+1E", - "fDX04iQckZjoY2qxEWTKmtrQFyoLKbpapG7uPfW7DfHVMISkzIWB/EpIqm2VhEtbZjTBbO4dWB1JHK31", - "XRVNiaD5rkEzs9SqDu7JSk2scanag7LXvNom662MgZ+jlNwuIrv9cmW9jLvwwz2kkvbGXi7IKmM4tIYc", - "38X1b8J6TRHtuHZe7+zKdjrZO2sZNwEI1UN7y27mg+4/jFsZDXv7Gz/99f/tfvzLn/yleCp2sySiG5Ex", - "3HdekHnX1BjWNnqvCl8L4EZambYFbBTBCTiNwgtinFQJviqPd7efC435a5wsTAEuihPK8n+vnNBf/9R8", - "zVoi43uQkytZ9qvBru4CSVhxdxytJURMHFy/iw6E6u3abrggc4lKcIVWpXGM+meZf1KuE3xu1MAeVLce", - "UUB9lQOmrVochiTV1oSFbaMwFsFB+tSrZVvYRBfNr+USBqxYm11VQ0X77K1JvR8wctk1PURdzXs7u3vA", - "R5SVKbm5sMS+RTcJ+k0FNTWVPV6jV1RC9okLsi69jNZIkqq5AwV2YbDr1wMMOMgb9N583zJaWv/ZbYDD", - "vl+KBvsdlnMt4zm4Aa1EclhY/0YIRn8c3VEdmMnsSVuirgokVLMOpeo2h9klWp0ZQrToYkicfmYiUS38", - "6SSr48BvJExtWLBlX/ZLBBWol8YeF7vMgRt04aPVIbVLFerSzEojaV6bE6c31quCNxPoVJPmckoEKS0E", - "fFAgxl6TZDYutEVOlYFETYno1msYmjIbgkKgaW7yOxLkscOLftDlgEYn+CrvAXzoWC7cNME8Cmi/zZc/", - "Q+mct66WHR27JmAYNa3ej05U5aI21fgXF6PMVYvzNu97N56VVUukX9PeqjFn0UeFNX38+HdM1QsuwA5o", - "zmC6c5AjsDEiIiCFuw5h1Ar/hyYkGvJMLd//tqaATV+KnCpfwC07mwcDE9sS1ytkgcuxKcbw0ac3SBJm", - "gqq5NpKtSjoiWBBxkJkND4SEjuDnomNAL/7yBbyFY0/A4kvCiKAhOjg9hv2YYAaqMvpwgmI6JuE8jIkF", - "n12AWwEl783hsbUzHcAf2A1UAeu5qtMHp8dQxFYYWyvo97Z6fdjMKWE4pcF+sN3bhJK+muFgihtQ7AD+", - "tLkIub1yHFk96Gfziv5K4IQoImSw/5snpl8RYYonSNA68aRkN6SYCms4pDFkGhhWofpbQLtyR+m+OY87", - "huCt3WRSzW3cJUnf2GX9qDnB7BqY4la/b0xSpuzBi4viphu/2/zMot9W+hyQxwP9tKDXO53SkvxLJ9jp", - "b15rPCvrkfq6fc9wpqZc0E8Ehrl7TSLcqNNjZoLBkYEVseEu5X0GLFTeYb991OslsyTBYu7IVdAq5bJJ", - "GSbadmfk0lbu+J2PesheAgDarZzyLNbSBJlId2fuKyx6k08Ii3BKZ2TA7DltastiAcZ8gvT5bMyW6tYw", - "XZvVz3MQf+bRvEbdvLkN3VzXuX4LAtdRNSUZAjjYsKksT+H0pYxBeU9JLARpXp9iMaYG6jHLkHsLUROG", - "mSrK+5pCzBdkbv3K3gZbofhogQfLQqDufw5Pv7XuT18BsFR/5tdR/gxZ8lbVCQaXMWGcRYXO5SKqsRjh", - "OPbCPExiPsKxrVd9QTwq6kt4wxKljCvrlBvGI2IwQtO5mnJm/s5GGVOZ+Xsk+KUkQqtAFmzc0toWa7Ws", - "ewmwYAkAfptSJrrPDTPEjc8XZP6lN2AHUeLK1EjzCY4lt4W8DVoSlciF4Bre9aPZNkR3HGZS8cSyFCvX", - "HTXD5JlKM2VvtiVRFiEdXoeytHJKogFTHH0WZEKlEvMvG5+LHr+A7UJwpPmk9IqZ0sZnGn1pGrUcYj37", - "Ibzqsf4IEGAQ6NNlEOi/JwJr2yWTU3BlSHBfTMpLupan3mu9cL1O4RAzlPLUwBYAU5n65JU2oNoEjmOk", - "YCu5b7W2CSvZMB+bieQrnWjTkEzeSG0bQRHF0mbq7zz17ydJQkF8Do7/OXvzGsFRpdfAvFa4jczVMtOn", - "KIoy0OSh996APcfhFBm9CaDpBgGNBkFuXUTrMNZM2jjpbhdU3J/00H4y3XRo9FOvp5sy2vM++u2zaWVf", - "76U0GSp+Qdgg+NJBpQcTqqbZKH/20U/QpmyOs4ogQGtG9q+7WkGAKlEcg+bcwCxC3MraeI4wKiRQ2Y8y", - "ogyLpYWOPKS3FNSmPJ7IMjE+D8CDOgj2B86HOgg6g4CwGfxmHa2D4IufAlaJbsZBM7WenK6dM9Fev7++", - "Os3S0tejQlde1Nvvy4L2tXVriodVuhYVDzM5B+KoV9BU7TLq1j1oPj/jyNWB+KHirVDxrOeipLzB9+Vz", - "wLBvTIyBW9PAtD0bOw1sqXVi2AJQTMHicEnRxuCgToMrmLdsftTN+UWzYqdpl4UwxNjx38498B/0W1S+", - "h36f3Ve/OAZI0rwO9ONiR1gsx4gdv0X8kqhvgeP69yVKLX7qQ/LvY+Gfl8TqfQXRatJsg8zcfZMf+gFS", - "PqRtxbysbdUzGFP3jDCFnsOvPfu/zuIBIOPzmE/O95EhYcwnKKbM3saVbov0oWhpCR+ZrI/8O5sE4nC3", - "1sz5+e9//gsGRdnk3//8l9amzV+w3TcMGArg9J5PCRZqRLA630e/EpJ2cUxnxE0GkDTJjIg52u6DmpkK", - "eOSpLSoHbMDeEpUJVrq1NBBY0jYIpgeD+VCWEWmzZvSLdGzxOYyD2WPCu71sSHmvO7qzGGJsZlCagD4V", - "HQ9AwjU1YMXW/gr83jMz54r/rO4rX/CYrpYvilwpw71dM8BrChggsW/fwQM7abR2dvZ8vYfAxjBcARgs", - "oDEXzVjlufdDJq2WSUaiVAUKUNnIplJZ+Ub/75F9p50D2Lb4PXmAm+rkN7uAjcuDCBI5ev2wFdq4g/10", - "c65hn3/2yOVKNjtobz7fchcumqiVIXx76+x4b5Hm5kmJZA9hAqM1F5PuSjyeHh67UkDrD8b093Jq6Jna", - "Ahr50YG4KSx5b2bZIWfjmIYKdd1YoIZDQnJTrcogj0UcvLWjRtjNq452WD7fNirgPY0nXY7jUxx5d396", - "1Dq9zjFSIDIWvPbjJFnFOkdUhlx/W+KWbohTW5/SqC/5Pi1z0SqHlAlxz4+cpeqSFc/HR25D3p9rynad", - "sfrZcA9C8agmEB9QENZq7pUwTB8TN7/PV9HhQizxXH1brNm/Py3ovr1YPjZ/TG6sqEY2LQUNknPjAfqS", - "qF/MG3e40LYHz8TPiHC72kFOw6zzaZlPUTgl4YWZEFxIL7d9j80r7Uxf0973ZPkCea6jsViS/1BRWhi7", - "Ba2WGbjHto7g3dm30MO1zNvbu+e1DOYhMgSbjJzH2pTow3LOwvXv6qr3Xk4zQ+xHeZidZnHsbjxmRCiU", - "l+sunwEbnyEsabVu73bb0uPg/dtXXcJCDnFoeQyVX4myT25ZwzcLZqbyg03a2IQmOZm686xJw/mK9Tfh", - "gigvB/9fWy9sQfj/2nphSsL/1/aBKQq/fmfM0r8v0XzfGvcjZj6tcNMq0UA0MSgNu0pDzd9qqaS6978r", - "PdVM+lqaak7XH8pqG2W1TK6l+qpdijvVWE0fD3QlkzObj9rwyMUnfmea6v16+SxHOrhmKqvXHrYeDxfg", - "54VHlKFMkkcYQElzjisfGy3d1cWGXHp8ONY9PuoAITuadICdZBNE7sl57cZx78qt7ff+PdcHyYhOMp7J", - "cu5JglU4JdImK8WkKoAfm9pdHM+Nivc3zKX9+zw67l2v/sH3d6Tx1xfUCG9zA7VK53dvtdX57fta5zcp", - "1DZ3zQI8dRz433pDUKFLom7LxpVc88VgR9+4fLYIeq8NlcJcQGBB7A/Yf2v74zdFcPLxJ5ckk/X7W3vw", - "O2Gzjz+5PBl24liFMCWoxWo9eH0E134TyD4HONciJa8+DlP/AVjPAdj8xxlIxc1newvJceEPC6mVhVQi", - "13ILya7F3ZpIVRCse7eRHL/5CG5BTH5YSfdhJclsPKYhJUwV5dEWgsRsdcVHmFvG7P1QKbijctC2tpLy", - "TblCAS3A+e89sOe4QCO8b+PI1QF4nDHyPLXA2tYcKQ7DZnvkW+OH/v0K5/u3Qx4zixmFf5F0qdYpfVU3", - "AekxyRQEJRYIIRD1iYTR2vMWe6godimzNOVCSYMWCQqwwZOfagXYhyxZBYv0oUMCBjAlsjNgUC9APza5", - "/BsXZG6wIClnOexjPlOL/+jLvapicT7oNrp9HcsPNNpKx7rnbWyhox9Ox3ow0XEvmtZxBZF/Ld8YYFCO", - "SL6TeZ7cRz9RNll/VBGoRljlcyvhGXlUrQ2ot2fRdTdkXte36aAtAezaapT/gSfu4iR9WrvDoi0REEUU", - "TxiXioYucbeOWf7jhG59Qi+nrJebx7aaqt+gf8HFRdsjzlPd6xGcdOUZfoO+BD08QAN7eJcCGNvmNNBM", - "c++n4ELJtodMwaD1czGMs0gfhO5AdKrkWPBkaH80eLV6V1g0UHBRhLbVhxY2uvd7cBi95grRJI2J1uJJ", - "hLqGm/RqWtXfgb5TWSpweD1hqLdNOSHGgNFJVyDIiki4XHMLtgb37IvL5ZWaMZ+sBsHIO3eIDx4UjAEz", - "oPTEIdifo1zIQgktEpNQocspDaeAiAFltaCuKoBV4DQ9zyGw1vfRS9ipZSQw6HxNEqENoZAzyWNigC5m", - "SXK+v4jY+uHkBD4yYBgGm/V8HzmU1vyAkPqtMsJFXnnotcXtWNOcJHgcmxU911ZjaX7rFvuigCgbMB8O", - "BiOXtkE6RuclSIzzBkwMJ1Bf8cmDaVudZmBJMxfFkQDCGd4kLAqaLmJo7EfD2Ox7q7K0ROYww7hjYI6F", - "wbzikxzUssLKOE3bsq8dJnDxLEmW8DBaK5VIlSrimfqrVBERAj623N3E3GgNh+YfCl9oRrXlffIis8B+", - "3utGgzLnJZUWqqVaNuZfsyQJOoEdTwmd7hra+wqEk3qDi9diemVKMCY/9O7rAJRUhX0JoaR2ctgi/M0q", - "91vzwnfvn7WEir4HL0v1PqsYBWVFGSYBddRdLaxHhXQAC7mgi5nyRL494mbZlaUSnu2utxaKf34DRuuq", - "W6+8kmNeZvK+r78WR/CYk2DkwmzGXNTT41fdi33zjHR7S7Iw1TYc8oM3r++ea8WYabakoicUJJXg54Mq", - "l4DrHE45lyW2H5EpnlEuLAK79brmnAkuC2M92ui5c82q59Z/e27V833ra0K4/Mj20YPPbcyd/wv3qPji", - "RcnaziV+x6nUgAIpEUYjQckYpTiTRGtLWUKQqTBigbwJDqeuZndvwN5NCbJVKksOhLyoMZXofDM576BR", - "plCMxQSsHfPQRNIJEvIkISwylWcHbErwjGpTTaAYK8LCeVcSqEQ8I0UBE2262xtKU/A6r3XaQa5ELjgY", - "zksFcM9RKggwkTGXWaXa7ICJjP3NIFfqZs/dQM8RkQqPYiqnea2IEEeEhV5YyLNvW4zdvhP3jKjFGrEP", - "cmd5I1n6kJeYZV9mXqX7m7jffGSBWly4+pYtxPwSpVc2m4bVyMezoi7uf+CWNnN1c3ygm5mcxMt28bdx", - "JVMpjP/jWkbZLRllpjtSLR7/3d61FNWcM1a5brE+2ZteuOSVEHIyX0vmbXx2fx7fwEf2jUjCTqNh34S5", - "XUz6WxC5lqo3krkP5By0vqSSV+wBRbAd1MOpT1yUpNw3IYbNhsulcVnmKIHBpuLshzCuC2MbHnBTYew8", - "rgsX4CXxTFk3jXGTXC5qx/sFsHUI/IdGv9ZmVxKEDy74ihuBexN2x7l4MwIvxfOY4+/9XibkQpiETluO", - "+PEAipV8gaULpjXwuHVyCdFx2SQfTk7Wm6SEUEtlhFCPWEJUy5qGiada45sZEYJGrnTk4cmRjV6lEomM", - "9dCbhEI9xwtCUigUQ3kmEWTm9vT8XGrrYhG8Sg5rJyBMiXnKKVMrR1G8ejeD+XKj0nn3LCctpOJ3f3kM", - "XvjHJ6RAdmh1xU5guRWpsGoMxnPBaZSZepda28IjnunWtWRxhXYncLaNaUzkXCqSmMi8cRbDJgLQXVuT", - "yX5nMko7iCqJ9H7oQAZeSkRCpaScyQGz5d9TInTf+nMo/lsEGXmd9wrnUvPUiL5vI4BND8bEbGHVRDWA", - "FoA6oMF+sIHTdAPKRfuDpOzwvmJILyAiDcl5MuIxDVFM2YVEazG9MEYHmkkU6z/Wl4a0DeG72644dfOd", - "pSl9zMbcW5TD8GzOzN9HElJVrLlLxEcn1l6S8mZx8gcW2i/W5Eq5JgiOu4omJE9+R5miMf1kRJ1uhEpF", - "Q5NXU6ReQhFmm305YCdECf0OFgSFPI5JqJxzZSMVPNwYZP3+dphSQCnZJjA4EHjNjxPo8fD0PbxnCkV3", - "Bkz/Axp+d3BqbmLH2PoISgNlRF1ycYGON96sCPI9AzL9B0fJmQkuzYH0LviP67vrZzY37iHZsEV5uswA", - "4ul3H8ZpNbgf3oLH6S0AaIl8NmsTgUNQiuU0UxG/ZH7PwIzHWaL/Yf44XgVQonA4/QCvfjParhnOym7c", - "BB/FprRziogpGvQgFxSGYI81vlQTzk0BlJhK5J73FDhQ3yN3375TvkzHb/Bq0lLUFeT6ZvbWfZ98dgwO", - "d6tMj8eyzQ2nuZkovtz7dIlps/fp55iHFxJlTNG4Amqg7TbAAdU/FriN9uIP1ATIjnSlxBG5SqkABJsa", - "PAIiesYSYaSISCjD8QbM2TQCCJTOi4VnnEKSchhTSBOjEUEpj2NA2bmcEob0bMBR5Roo3dNKWwGi/E75", - "ilFxNCIhT4hD5Vz3mW5/x1S94KIKsfmtyMV3Jfrr+eip6nmuQBVt7vGrUEZP8BWENUeZvSZ2I1p7yYsf", - "jSuog2BtBsF2Xw6CDhoEW8kg0CtwiMGFihXaRQllmSKyh46MfwvSUPf6SJKQs0g6cFDnwdvuy6akVMOW", - "DRmOe/Ddfao9lquAlG9tJz7xoN9D+ntIsEFr5Q1n92TUgU0XIZ4pCOB2+8q+FREF7pH1e7+BLe2RH7Z9", - "G0n+d7t9KzIKVlmLy9LSG8mew0eu9Lq5pIoplwXqJApxikOq5h2E45iHhfcgk/ntQDcfykgQfKFtqN6A", - "vc2BK20iBDo8fd9xTjMUUXlhWrB+sR56MyNCZqN8cAikgfHgwWKQaMAURyGOwyzWfEvGYxJCDkNME6pk", - "g18tH8pdlkEsOvEsvHuYw9Y8LmeSnydg9Qq2kDWO2zBLvSFIGGOalJ1KdeKA6gtXuuD2HelGuT6Gx7G9", - "3goFlxLZprokphM6iu1ljeyhd1rlwAkZsDTGjBGBMmnijvTQu6kgUmYmMUY3AHVmDUd1UAF0kgqurJs4", - "5lxI49nVHP7hBElF0iVs9ta0fAJzviOYYNO47emBDIbaGJqPJfsK0gtiOMUQXPORPqYfINjHDOih4YQf", - "y8Z/J+hkQoTeFdgIWXM1ara1I6fZ9JVMj0aM/LP8rXYY+XmrpWjuUqTzUqCKoXtxCAr0dW5gPZ1f0EYs", - "E/voetkXv+qPWvZdjfL3D8I++spZfi+lx85KwdVtkfULDn9sIPelkVe2aiVBYTUcQeuMhLvMEGiNO/Bg", - "cAOPGWUAV9IOmuAEvj1G6N9vdtx9w2w/bt6qoARUCus0pEqthu/8JjjwbnA7Hzg79Aa4nd9UvhLgLj5c", - "3ug3lalU8QO64iHfPTLnXSUoGXhOgLFoSlAyUs8GEiw1lD7Yd9qZSbbF70mDt3fP19DfHdl/WP0tTIYS", - "sfwuO5Mb7XBbSJKqubtc5OPaBaCknyAZwwf8kMcQ3B3ewg2u12+PPRyfNl6u/6indW/390XR4eOjx19E", - "q7znKgfLhj51uliEUzojzU736g62JEoF6aY8hcuVyBDM0sOdZQqL3uQTss1brCr7L0QdxDGJUEQFCVU8", - "R5QpDhLB9PFniQTXlgA852Luc6aXd+4LwZMDO5sV56HdU9YZVtz5JvNuhBXuzpy0WeJC+4qbdne3rQUe", - "ogy9/BmtkSslDOIuGmvLB9FxTlJyFRISSeDJ9fKAN/sNnk36iQwnozajXIKd/MZiU6Mwk4onbu2Pj9Aa", - "FFuYEKbXQqv6Y9BkU8FnNDKFSAuiznhsqLrZQNDr+l21UpFXynDGhRncg+gwbQ6kySeaVsWCCV0I9oMR", - "ZRgGtxKluLqnTEKV7g9TSGso9o7jnODHEWYtvzVn7GhO1EaOI6Li3EDjrf845h7zMVcOTHVnWuW0a1cq", - "sl2sassQ0rsAzM3jmO/Xbf3h2wmvpPJRRlZa1/ksN0ib3ObfFgv27+98uG93+YdHHI7/kjjju+QqhwZ0", - "iz6GecVDHKOIzEjMU6giad4NOkEm4mA/mCqV7m9sxPq9KZdq/2n/aT/48vHL/w0AAP//ZNbJHlB0AQA=", + "H4sIAAAAAAAC/+y9/XIbOZIH+CqIutkYaYakqA/LtjY69tSS7da21dZZtud2mj4KrAJJjKqAagBFiXY4", + "4v7aB5i4J5wnuUACqC+iyKKsD2vs3YkZmVWFj0QikZnI/OXnIORJyhlhSgYHnwMZTkmC4c9DpXA4/cDj", + "LCFvyR8ZkUr/nAqeEqEogZcSnjE1TLGa6n9FRIaCpopyFhwEZ1hN0dWUCIJm0AqSU57FERoRBN+RKOgE", + "5BonaUyCg2ArYWorwgoHnUDNU/2TVIKySfClEwiCI87iuelmjLNYBQdjHEvSqXV7qptGWCL9SRe+ydsb", + "cR4TzIIv0OIfGRUkCg5+L0/jY/4yH/2DhEp3fpgpfq4wi0bzMx7TcL442deUZdfQG8KZ4glWNETSfINS", + "+AiNsCQR4gzhUNEZQZSNeMYi9O7oDIWcMRLqxuSA8ZEkYkYiNBY8QWpK0JRLBe8ogcNLpPAoJr0BCzq1", + "9SBMP4lWU+lvU6KmRHgGSyWyraAxF0hNqUSU6ach6ZUXTImMLFK2E9AoJkNFE8IztUioX/gVijmbwLRc", + "uyjJpEJTPCPoExEc/ZHhmI7nlE2aiTQiYy4I+mWekgQzlMY4JBJRhShT3M3G0KjgsSeJj7nohHFBhhGR", + "ijKs2x+mXJgdUR39G/gDx6j0LgwN3kdqipXjcsYVuiQkrU4UX+HLKhl/39npPO/3+x87AVUkMdsKX9Mk", + "S4KD/SdPdp90goQy8+/tfPSUKTIhQg/f/oKFwPPSdCTPREiGIY3EspmEMSVMoaOT47c3nECw3e/B/289", + "CzrB9vOd3vb+M/j39n5QntYC4asj/7J8650rrDK5KIPMbhpaRhmWmGRx1r9lyYgIxMcozIQgTMVzBFuK", + "RC2YrjLtvm8pQs7GdJIJtwV9W65CzimWCDMjNLo1eVE01mrfhVqIRfyKDQVJMGWaxguDeOseIb1Dkd1E", + "ekghZ0rwONZCQSmSpEq6XdTRYpwhnKYxDUH0VDbVXtKXQSdgWRzrh7URFqtNYjqh8EIr0lBZWiT3LVIc", + "EaaIyHd4G9JUxGJTxwW5vatRyMX2UlBSFvqny+o0T7SEFyQ0081PgApFRiTkCUG66eoK7PR39rv9vW5/", + "/93204P+3kH/yd+DTjDmIsEqOAgirEhXL3ibZVouv48KKukXkX2xOKo8tOvVZHA7domxVPmuhk1O1XyI", + "PWN6RxMiFU5SvbH1GErEbNrWrsH6OjjKLyXw9lcRmJFrNbQU8s7Hxx/kOiWhPmK42575ia3b6yA6Rhjl", + "MkCzqxGMSyfy/KsmIgiWesBa79Cn0+9BxmSW6rOQRMM0xkq3q5UUYINhQqXUn+Y/RFSajdkJHJMPGVdD", + "kTFmXmREXXFxWX7TtjKkadAJplgOZ5M0CzrLzoEqU0MXJMaphPbsioshEYKLwOia8+GYC7dI+hArSLik", + "qQUKyfzM8lAo6AQVAuTy0c3FjTtfVe/goBfgJWHUdKNXw2QWB15ua3G4+dCWS0ojlo1W6pYZ2Y9lVQJE", + "FE8Yl4qGspXchNNYL2/CI4/oPM6bQzQiTNExJcIqqgSJjMGx5hpBuhFEGcpkbR/kuvSQzLTxM5ztDVWY", + "LhKlZimUF6902BdHTOmYy5c/3ykrmLQ6d68lMsMU9uQxmVFztFSVIbs0w0jQGREe8Z2fqEYUmvfQht7r", + "WoQwzshmhVJsRiOK24iDCMY0pB7uOTs6QeYxOjlGG1NyXe1k5+noWdDcJMOJhxd+yRLMunpD6GG59uHd", + "ctuv97w6P0+SbDgRPEsXWz55c3r6HsFDxEBlLLf4bMen+qUhHeIoEkRK//zdw/LY+v1+/wDvHPT7vb5v", + "lDPCIi4aSWoe+0m63Y/IkiZbkdS2v0DS3z6cHJ8coiMuUi7ACFq5ccrkKc+rzDbVVfHx/88ZjaNFrh/p", + "n4kY5oeIj2AnTo06OXZ6gv0OfThFG1qGRGSUTSaUTTbb8HvINTn0Uec7xGGoyL6jzUTltJQbn7ehIHhF", + "d/qNVp0tbrXMrOQwkU2tu1e0RE1oHFNJQs4iWe6DMrW/1zyZ0oYxJ9RCVy/0zyghUuIJQRvgUgHzwwhT", + "rdiMMY1JtNlOmW2azD/4qHSEVNgb2KKLR+H2zq5XdiR4QoYRnVifWP2I0r9rFtPtKARv+ycCh3m7eUCX", + "gowX+3sJohs6EWRMBNE8/pXdpYLPCMPWevkT9Bv8H1uFs3DLegq3gJhnxetfOsEfGcnIMOWSmhEuSC77", + "RLMRkBrBF/4xw6Nla13iKKmwWL4/4I1b2ImFXreSNtZtoVUbPFn5yTv9Tl12gmjMdYmSFGgUkS+0UuPR", + "DjhT9kHNfcknKKbMWBxatTNrAXrVPCU/xRxE4i3RISf/4ubX476B8DI/NLSmn3VyBTzmkzI1pwQLNSIV", + "YjYcYbahYnSN5D+rbJ/aWYUlGS6XIGeUMRKBv9hubPOmVmO9ZgbsokuqhjMipHfPwbB+pQrZNxqbinl4", + "OaYxGU6xnFoHWxRR4yw8q8zEo61VHPEY7HHXIGgRYL+e/3K482Qf2Q48NLSeS/3C4kxKX+vmzbtIYTHC", + "cezljWZ2W/+MXuQQPwcUzsqmsyfnQMeYRtIFdjWtnZzJqfkLZLceFZx9Wgxo9or13x89kz4CIWGshMbb", + "G78OmHuGJzHXNJ2jjNE/soqC3UMnY3AQ64OCRiTqIAwPwO+g7b8JYURoOVV4hkpKMNogvUmvgwZaL+xq", + "LbiLd7r9frc/CKpqbLzXNeZ9ipUiQg/w//kddz8ddv/e7z7/WPw57HU//vVPPgZoq5k7rdDOc8Pt/Q5y", + "gy2r6/WBrlLlbyz9y8P3SRyz1CdaTqy70kcni4qDmWvEw0siepRvxXQksJhvsQll1wcxVkSq6syXv3ur", + "tIB5LCECm2gyrUmGmtEDbLwR8ysiQi2BY6IZT3a0EKZKdhDWdjMIL6RPyf9EIWZ6LxjlggtEWISuqJoi", + "DO9VqZXMuzilXWqGGnSCBF+/JmyipsHB/u4Cn2sm37B/dD/+xf20+V9eVhdZTDxM/pZnirIJgsflaz03", + "hvyKZtmKOOpmMah5CWUn5rPtxTuor1thN5FlK22Mucal1kIod5GtGMji/a42thKP6fBmRoSgkTuWj06P", + "0UZML4ndL0hkDA2yfn83hBfgT2J/CXmSYBaZ3zZ76E1ClT4Os+KUN1e2tds1Ek45KCpxzNe5TgNNEQwc", + "HC89x5eRxkvto7zdxVP/Fy5VN8EMTwiYo/ZFNBL8kuiBmjsBSiS6JHOt5czRRDfanVEJNzyEzdAMG69D", + "b8DeTbkk5hX3SIJvn84ISnh4aa5+pxws+RmOMyI76GqqVQ7wCRIc25+RuRgbsKkepAx5SiJthJjXYGro", + "grDZBUpwCtscCwJ7HCVYEUFxTD+ZK3y4ZSAR1SfcgBHYGCjFes+HIRcR3LBxRHA4LVHhzxJdGIXlApq/", + "oEyz9YXZmLXL6s/Bm/fvfn7z/rfj4ZuzF78dngx/ffE/+mfzUXDw++fAhGrkmsrPBAsi0J8+w3y/GPU2", + "IiI4CA4zNeWCfjLemi+dQNNAav7CKe3xlDBMeyFPgk7wl/I/P3756BQy48ae6W3gGdgXrzJkzlKPSDp2", + "3kCJrIfJ3W1okmkR9ers/ZY+nVMspZoKnk2m1Y1hVYO1tkRE5eWQ8uEo9Y2Jykt0svUGacUFxVRv0FxR", + "2e73T3/ekoNA/+OJ+8dmDx2bXQvD1zKIC6s/yalmnzzq4+jsPcJxzEPrQxk3XfC6rnwCnjAl5imnPiOu", + "JpyKVxdlVLdbPF1DFG2NKNuSehm64Xp0B765sSnxgs2o4CzR5twMC6rPaVndK7+9OX4xfPHbh+BAHwRR", + "Flqv5Nmbt++Cg2C33+8HPgbVHLRCBr46e29uPc22UWmcTYaSfvKoEof5/FBCEi6MCW2/QRvTqqZh9i2C", + "xRkEu69+Nsy1/Qr4yi2KvSPKWzEN1671Xv3s45bpPCViRqXPz/ZL/syt/GK4T4W3zS1ZzrTAxb2S/RLG", + "PIu6pS47wZgKEkJ4hf7XHyTRivzsU/VayvOd3/3VSoFdoZniOKWMLFFNvxEV8YqLy5jjqLt9yxqivVD1", + "hMaYB9X1zW/WHEssRJyNMIuuaKSmw4hfMT1kj1y1T1D+ci5cr/VMcPyv//3nh9PCztp+NUqtpN3eefKV", + "krYmW3XTXh9KPpEs9U/jfeqfxIfTf/3vP91MHnYSRhG5kVJn1/+FaaEeNGNjCY07tOFmOD+984AVxa1B", + "DZ8jx3srr4F9gprPiIjxvCR47ZiC7T5Iv9qoBIUoSWS/02L0EumPV4hh3Zo75F/Vjfydvl/QegblGdPP", + "WlbYc6HNSPKBbO+c2j93FofUMKJLmg5Bax7iSe7zXRYSen5JU6uKwxdmGePYCIIoA+V9xLnqDZiJUNFr", + "BwtMrkkIMk8qrNDh2YlEVzSOwUMEQmXxaNGKfSm0CV6XSv+3yFgHjTKltXWuCLJ2E3SSwVjg5RFBGcPu", + "PrymO9sJLoYXAFkuiWAkHhrdWLakjPkI2Y8aiQNTHWNpQ9SEytIqvY5/PT1HG8dzhhMaol9Nq6c8ymKC", + "zk10wWaVep0BSwWEKehO9H6mtl8+RjxTXT7uKkGIG2ICjeU+NntZO3t19t5e98vN3oC9JZqwhEU20Ned", + "ODYINOLsz3rHkqjabLn/GtGbQjokw6mccjVM8+DpZdLp3L5emOLtnQmdYBamWXVJdzqNQaAzKlSGYy1r", + "K+qk94LfBLF7zAYTI182X6zcK4JmVfVmtq3HxbQMEe3ecFmP48RoSq0dJyVTfsGF4uzMz+0Gu6L9E+YG", + "stRxVJiaX9HXuWlkIXjH/NxxM7sBlU5ymtTcTbdDnkNZMs1bBZ+bGCyjEUq0caGtecvH2n6/6KCLv1R+", + "0HvfmRZav7hChhogT5j+qdx+3Smx0l2wVrh3eXGwvPl6HMrGSCc020ZKYCZNjNoUp6SHfgEhjhRJUi3J", + "2ARRifLQLsT41X8ibpQa9+mA6aFJEydiyZE7jSSdMMomm1rN1wcTjiLjWRpnKhP6vRmVBTWrrOO8NwtR", + "rWZ0xMhjyJCgLIyziKAL5+G5qOqFi/6fRZPQOoQWLBxDErBswNhTW0mmdPd6wglW4VTTiWfKBI7ZqVeD", + "+mpeplUXqnYs+VXbDdb/PBcX9USYmcfE0ZOzlzzgFiz5J5vcgFZR8bsoL8kclty5I/GCQ7LsifT7CwWR", + "PJ4Re+yWfZkjSPXhRnEq3JjGIWl9kHr715NcfN65VUuh6dWa/FVTwZPiI1XXTbbgGKv9u5hwJ4X05Ex/", + "HW0YSwLEB9PjAIE6dtExthIBDwRimlliFFFBQrXQPGWTAYMYkgv7S8+2dqE3udZRbiVxCvIQQGkvLy0q", + "raxT+6AZPTWeUKVI1KnqBpeEpHL1pLR6bR3XHu+6IFeCOkHmgopbqmeEjbkISWKNhK8zHF+UGvOaces1", + "sRjSYehbGrPLz4DsFBKZ+CGzHuBmraRt1LMXo5rVZkIIql1e4Di+QBv2pU0kyD8gEt+uFeOsYPZ3R2eO", + "BfJr7w+nHc2RWgpcTJVKh/q/5FDv4ot6Y/Zbt8OLzLJnfbCv9vZ27apap5sZcK3Zqn/NGxbRvDRO/W68", + "WdN8oUdp40zaqPJHxSeFJ/WSsqhtA7/qdxu9c7li5CyNu3bQpYJ0s3QiMITY3qZ77sb3pkDNZgm+Io/X", + "FyZZZAhmUvGkHG+/UQvxoNVgkCqxZjzuRlhhcGW29Lea4S4GHidz05SxxZo8McPJyBM3RD9BKsCETvBo", + "rqr3B9vebL6vvcR2Y/EtS1MAv7EgSTRUfHkIMx0j926biEWTb6D4cDamfHl6h41/qeTfmePI2rW6iW4a", + "UutOAB0nnJoIU0MEUBo/nJbv7noD1oXj9wAd5x3kzeZNYtAtcWRuTja4KA3CJHKg0XwTYfThtIfe5aP9", + "s0TaYJkRl9EwxRKNCGEoA9cznIZdcxaXB5BJODRV/XPrOzHJD5twRcnts16ecwxemjyDGkKlRrQ2H5M5", + "CQtl74QxK3vBWnmtlgV+vyUTKpWohX2jjbcvj3Z3d5/X/Zc7T7r97e72k3fb/YO+/s/f20eI335+h6+t", + "w6psscFnZelz9P7keMc6S6v9qE97+Pmz62usnu/TK/n8UzISk3/s4nvJAPGLsuMiag5tZJKIrhOTmqt8", + "sXKlkLSGWLgbh7jdUcRaEYC77F1DiXf6zbtIbfEFTduQ3fWTT+oCc2XYdWlyi5b8PAW7s9glJQ3ORjeG", + "1BvHeUzl5c+C4EtI2Vs8txM8IXJozjN/PEMmTZANubbeDcG5Gktzb1r1em7vPd17tru/96zf92R0LDI8", + "D+kw1CdQqwG8OTpBMZ4TgeAbtAEXXhEaxXxUZfQnu/vPnvafb++0HYe54mlHh9zwcl+hDUuRvzqcEvek", + "Mqidnaf7u7u7/f39nb1Wo7L+4laDcr7likrydPfp3vaznb1WVPAp9C9chk1dgfdlVh6a7H79r65MSUjH", + "NESQo4P0B2gjgSOM5LdV1T05wpHLP/WfHQrTWC6NmDCd2TeNoy3JYkXTmJhnsCCtfNEw82NoyYuQwVie", + "77teSzYvaWWEgJtL/gqq5JdVSHdqEppLyhMlcXRgduhKOQerWQzsYxMf2Dm05IbX2nTqxmRG4jITmKPL", + "ZNYKgnI+MYtWmRVlMxzTaEhZmnlZopGULzMBuqhpFOERz5S5ZrQJ2kUnEPUMtsdYi+t2du5LLi5Xxo/q", + "kzjPQ1/pFToER/rYumrgFMfIfu1SFEpKX34daC5N7XOJ3povjIeo+DnNqqg2HejJepIYEkQqDpLUOgxt", + "M221S7/eAs5SF/5h+itk5z3FvnTHJlzgNi1sOHWG5ha+xdUvBGWUl8OS4M+y8DYkJOlCigvMytELXiU4", + "ArQZnhoECgU3r7B7ZblZxdEIzFMbhww36aSH/gZ+I6xQTLBUiDPbttVGoMcBK8/pJ+ArKhGO6YzUWUkP", + "NebhJYkOzIJsWe7ZikhMFLF+5QGznwiiMsHQXv85ypjSuzGOkcxMiOalhABZ82WENiQhyG2snn5u0AaM", + "6zJ/kJBkaIaw2UMvoZXybFBlMsZ5qnvJGB6PSahf0xyW4PmAjUj+KZ5gTW8zXc26pgs79bZbQWExIQDO", + "oVaqs3ro7+D9c3i9da6C/nCll63FpmTk6j52JCRzdLVM62qOv3WHF1B8WaRi7ogqXgIVTdCI9BCIXgiZ", + "csmjNTF8rniaak6wJO8NWL7J7E/SXK/BznJ7mArEBZ3QasdV7+tdhjyuw4qOm27MjuUPF80XeGgkUuOJ", + "gMfKAHFcunw6Uk5us4sQdILzHLbEHlNV0rzNoV8WKFLE4S4M8dXZ+3UDF1PBtcxebAsCZexTa7a7kL7X", + "e/3z7vb/ZcJzNb+B/k6ZCa5JeFRDGbHvt1NLXp29P2saU477gcqjW5hTHg61DPnMUcTeONor64oIhku5", + "opPCMHvuM3TGAidklI3HRAwTj+f1pX6OzAsm7o0ydPpz1djRRlVbl8pZZXHApzLGoYVtaEd9j7e2No1O", + "iZof/cv1lpjTpSnZUy+VsO/YfM8e+i1HWkGvzt5LVISwedy41eVtTKY4m84lDXFsWjS525SVva/AnK3N", + "p7PiQ+un9hhRfoAetxHQxmySZrANz992T9582EoiMutUxgRhZ1MeEz3uzZK0mLmUzyLzoyIkZk1uMMMY", + "su0GKtEq38GtiVTarx7qKK5wPJQx90XyvNMPETxEGx9empQ8PYIOSitLqX8vUaHC3/veHaMlUlO359Bh", + "3Z9e2eBex0IVKtX43krTq3Tq2yq/EBwbhNgqPy+iY/HL6kLzy9WITKYRX78nLmugZvH6MvuOTo+NwhBy", + "pjBlRKCEKGzxaEvxT6AOBZ2gq8+oCJME4jDH/7k89Knhfqacqtfo4T9aAHW5E+9+AxjBWxOfojVuRsdE", + "KgtGUOlZTvHOk/0DA5kSkfHek/1er7duAtOLImOp1VJsmfyOUi5TT06/bh3uIE+pzVw+B2eH734JDoKt", + "TIqtmIc43pIjyg5K/87/WTyAP8w/R5R585taoezQ8QK6TvW+W59Z5veDEiCqA39sBXrot2cg7AWSKr3J", + "6ApPtH1iOO5rs85vjEtTgKOpEh5NOVq4BTYN/bTcTe4UI3jH9mlM7hy2Z9FBfiPgJbkUm2IBlyIlLEej", + "iGPzV8jZTO8KHzRFRYC7Z191uWRDoIYR9XDy36y1ZyJoIOVu9X4LtnCarmZbv6KYy7+2kDw2cd5zEj24", + "1L/JBWy19zeT//7j/5ZnT/+x/cfrDx/+Z/bqv49/o//zIT5781XpdcsxEx4U+ODWsA7g1rECeNCWlU6x", + "Cj0K1ZRL1UBh+wQpboJ5e+gIDL+DAeui11QRgeMDNAhq8eODAG2Qaxwq8xXiDOmmbBrMpv74zLh/9Mef", + "nW35pd5GZPNdhF2QPM1NZqOIJ5iyzQEbMNsWchOREPCh/4pQiFOVCaJXT+uw8RyNBGC+W/O86LyDPuM0", + "/bI5YGDhkmsl9AxSLFQO8uJ6AKawozIBJfZ1EjnUAGMhD1h+LuWgAcZH08udIHBxUw/H9RPFa75wUc3T", + "etb3wStASKBeyJhKRSBqP+dszUZ5rCJ61q+Iimf9Z/2VCn7OQ0vYD3bCYjEIx5Qt9pJhYOjaCG4IX2xx", + "0aJlk9kj6Jd37840GfT/niPXUEGLfImNkWcCRKXxEapYlkJDN4NmZ3DLCRknGXwWt0gpe2Fih9+9PkeK", + "iMRlc2yEmpxjGur5QWwIlTLTrEgxOjw6fbHZa1HNAmibj3/JOr7LZ1jP/LFOsyZfYM7xmr4ddHIMsdt2", + "hxYKHMRcveQCxUbAFPv6AL2XpBoIDUtlQj7MSsbzwvNmToBBsOlaTOuS4gC9zfVGnA+lUkGj6swr9iU0", + "a2/lTEDYQuudBWx64ewiK9og/AurPINAn7jNomD59vdQHPa8Dfov+TTX29tlZ6juzM8axdrfNqTO7as7", + "u+sauevCf1QzdEvZ3TkCSHvojruAwFg0+K6pGjaGeCD92AZ0OLPmwymaYsn+rOBhzbjZ3n3aCsxV99o2", + "OKIcFsHHZkj5tnTpvvmlvkl8vqRxbGJlJJ0wHKPnaOP85NWvJ69fb6IuevPmtL4Uy77wrU9xYbnMex7T", + "GbF3n7ZYirkgLuU2e6+GtQBfuKbtDdjfieBwZNXc8lAMh5EZETaMVetLWCKMxph1eQbqDGGqh966Okc1", + "aeMNaW4Bd+IEwKuz95AwhuXQTac5bhgXUybXVCq5mPLd6gJ2ObzKLxUIFG8O/eYt4qK4AI6FadwH4slD", + "RsbeG9pKEQvgOfggjmE8RqVIAguo1kf/+n//P0R7pFcNjShvMBuDYeMgSvEBVA5YgXBjAixMan5zQERp", + "/UshEc17b81Ymxb2bwUQ5mtRXawNdEegLo1Hpg8QpXp6mp9vF57lToazsqJQWVV0eRo3xkPpBNQTo34o", + "9cFGInRyVuCqFr5K13xtTrY813a/39vut/HcJjhc0vfp4VH7zvs7xj91gEcHYXRAxl/hObaMbXR6HF/h", + "uUQDZ3UNAmPmley7kpyyllmrW/lF2JmboczU1cRVODLr4Ma0A4T5WhSOZejq51Vc9daq+1cUH2oVGON0", + "GRsSY78arnMpQlDIszjS6vFIb11jrpPIehUkUQVkPez29+yS8StWnbrxjWsB8EdGxBx9OD2t3KQIMraI", + "3C0mDqE0DevA07WWYWeFBbVyNDfEZrkPPJa62C0dd7eOvlJ25bq4bcOhLVy6hb7tDaegzCyN5pMlc6o5", + "4yIyG2aZT43Uj1y21vv3J8cV5sB4f/tZ/9nz7rPR9n53L+pvd/H27n535wnuj3fDp7sNNTHah1PdPEKq", + "upubsyOB8ODYNsmv0YHeb3mI0yhTKA9/1Bv5SOvjqKT4m1xA8DWdMKoA95WyiW4GXD/WLjBJ3QaaljKq", + "AEUEMKwo01MGH5tuxAa1HaBX8C48wgnkKLpBaJO36l7C0dy417VgcF2n8K/lQz6fZlDhC76R00whqAin", + "p63JYO2z5U0YGXOAfuPwjXCB6YzXDT3zOnh6Fl+vG4UbNtrMhaxDZ1ZgHqCXuZDMxawVqxBaXJLdNpsC", + "MkU2KyGZdsUDzS3FypWiDTuBoWjQCRyhICpxMT7Rjsubl1VmRd+9E8ExiNAi/itTNLbAKDATCjXRYCIY", + "FrdpJ1sQQBJZw6bpFtkEFVk1If/ICYoPp2gDUqD/iqwVrf+1md84l3fl3s7zvef7T3ee77dKdCoGuFrA", + "H0HI2+LgVkr7MM2GrtxQw9SPzt6bqqYhZzJLjO/Hzr0UOpwKHmptlTJU1C8qOn/ee17O74p4Zmq52SHZ", + "ZNAvpYqFS4tNNVyb/kHjGR2P2R+fwsudfwiabF/vy52R17grSiN6NeGTsgN9wWwko64Bbm0wvzVDCdmY", + "pfaWSJgBOicKAf90EQ7hkM4j1SzLuVw2S3EvY+3t7u4+e/pkpxVf2dGVNs4Q7NfFUZ7aEZS2GLyJNt6e", + "n6OtEsOZNl34LmDKMKvA+fdZ7nKoFv/tbfd3fVzSoC8VXGPbniWNJP9glSA7KUt0CLjLFaSFXe6l9u5u", + "/+nek2dP2m1jV2pTXC+XMA7Hx5DHQh+VV34DLl3eHZ4h3boY47BqoWzv7O492X/6bK1RqbVGBbBdBm5n", + "jYE9e7r/ZG93Z7tduqXvYsUmElc2bFV2eTadhyk8q+EhxaLo7TSdFj7F0zDYWxLGmCaHoQuKqp0+BlZn", + "KMxrxSK0ORisk2Dh4GrxbSsTrVYrzKgGXKBSkdXeav/vzdy5zWLanAerxfiiDh1jpsllUz8MeusNaJcK", + "MqM8k7fQEFeQcjUcx5yLtb5tijJ7S2QWK+OCpBJ9OP0zCBHNXEgqklYzKCz7LUmQueHk1trAFZ7wc3UT", + "sVqtRpulXzbhTsM27SyLjq5s/8Y8tEiLqoytjmg4wnGYAV4hztdTzwoySnimIP5ibmJ/4phzhsIpZhMC", + "9R8MOiqbIIymPI56gf9uKI6GY++dDb9CMTfwKpeEpBbKzwxCf2ZrYaONV7yoYmlYqQbJ/SQxUsWCtdVL", + "lwdLy2A35K1pemLFS8gf5pOKNR/ziQQrUEFUU68OOJViYYKVMDPQlLPEGI/VhLwdfdp7hliT3r4j1Byd", + "fGwtWqtjKJ5TEoeCS1nU4v9wWh3msrDUhDKaaDm7OkqhOtgWrCtTzqQvYMFWVm6LYOs7ED3xfl9zJAIP", + "Q1jvsrLvFro0wSwDcL8SI5PrlArDHu1CHqZcqmGeZLTmYKUaAnBbJkiRiejOyymkdcyNiIN3vOeiE203", + "IVde0fwGXy9wlb+ppgE2y1QvRf3U6uQ86GPjxTSrpZldRapYPS9onUTAAumLSmiVlnLQ0AbjqiKWSmhV", + "m20uqvw2qu6nqYr0673+edscveUpeWdYTU/YmHvgYNdw+NtEBxeRkhKRUIAuRBFhFLLW31Q8/9a3BakT", + "sSQoyoilnFFIBbYEx2Z7A+wrc04xyiY1WV/vsI0b3oxhOa4b9GtfbHPlKP0B9/Ze3t6KS4SL0PtWV+BU", + "Dv2e4sWGBZlkMRaonoe6ZMhynsSUXbZpXc6TEY9piPQH9eucMY9jfjXUj6RBIdhsNTv9wbCIHK1dz5jB", + "2bhhsyC1fospQITQZi1rAVwvW+b7Lf19qxtcbzTYSxoTm6r5ntHrEqNXgY/2dvpNCS0NjVZSWRbTfNeV", + "3JZlfTveZeAe5oVSPEkWJuSpditRdURW5uubLQQOLkvfWXTFoA13KeyApap0LQE8tfKEtItdrIc/uNFs", + "SRJWe9979uTpfkuEra/ydS4ppP4Vns1ZssSj2bBSp23cZs+ePHv+fHfvyfOdtRxULlKmYX2aomXK61Or", + "h1Rzmj3pw/+tNSgTK+MfUkO8THVAldpGNx7QlyVbt0ieb7j2aIT2j8sr6e5Zqh7Qdj7GJdrSYUXlKpXv", + "2yCARkNnZGjo1i0GU0u6aDWGEKc4pGrucZjgK1PlIX+llgTexptWHayHpLZti+OhJZfMRkWY7obrHP3F", + "uNZrvPCsNVCfzEZNbvw39V6NE7/wAZWviFrc0BS1RBbdBfl8rrCsRHXov0OAaS/KM9bjh8wb7QvRO17P", + "a9EXoaA+IAN/3fny8teWs+T2rSjJdYovO0Kbt+BaNrTnRPZVpl0dhlyTD/YAvNlXw1EZQnMpRmkFb7M4", + "ddfvt11hycXvCmSy9forhYCu82EdTRD40Y7Bkrxou1NhiQZuUlysBpG/A9gnE1NwI+AnG45wL9hP9uc7", + "wXtaWI5zoty759qiz+IlGPBMETHDHseUawK5V6p+VCOJO8i6+NB2slmrTro39etqNjG7ZfiY1gKHqSBj", + "er2EW8wL5riuxstLS4GoWidAoo0EX6O9pyicYiFrY2d0MlXxvOpk3fPkxHxd3XaitOrcvqJCsZruw8Ub", + "Dbuc5dZ9W/a8lNzhr/RAouEy+IKj/DXnM07xHHTLRkPw6e5ev7+7078RfsFtFaAotdMUEVr6zjpzKleP", + "5Rby+M9FlNIrQU0dQ0cmqQTByQFEU6U4JCgmY8juy9GhV9r0C10vH7y9JLWB/zn/u4VyKRPWz2JFHOMM", + "FA/XjoV+cNMI3C1tNbml/Hxx2EtSAHMxEy7kAtbjV/e7/d1uf//d9u7Bk/2D7e27ADzIidQUwvP00/bV", + "03gHj/fiZ/Onf2xPn052kl1vusAd1DqplQ6tlT6xc0iJqAOX1mGbJYkpI12Zh72tDkBeIgvMTdLK/b+e", + "98HMYKmycF6dZFlnwKogTr0K431kctnRL3Wh1Id/crx82DeKI6sPxM9g9aEAP7UbDADwbH8t2kvGWp47", + "70svtj55lsY2rjp7fEHfsLW9q9xAcR8/VwRjZYctO7EXTzWPCTfhgqppsvx4yF/LsSPgMvyTVFE1kaaH", + "TiYMwKbLP+d3H+V68PrjoBPEn/aqe8b+3j6lymIllHNq9VKX1YAWdwOAZb6cCvBKYVoIE56ABQFC/LTd", + "3X4ON/Txp72f+t3nPfS3UqRAx1CrTL5t93bl134bGuaCEvROc3O+/Xyta3RHz2Uc9Ks9l5oOYouiYHm8", + "AHN1Z4ULm64scPF4YY1rWUV3VlvMnmbDspYUkRjPffVZYjxHIzIGQPuafVhmMjQiE8pkB5FrJ3QgH7ts", + "CtnK3bt9OQgQF2gQPEkGQQ8dWuwRsFYLIPdK84DSXOITmtiychanuTksZSdplz9RNx7WA6RyX3nUs55f", + "P3v+rr+9NiDVesdk7yuiqr/K3G1n4sZYqibb4jWWKrdJkcishdGByl5s3vNVU7DpEBDeCigSB0jWvByu", + "jIQsFD/nCemgCVeoSIRYqenB8EXGvPxQHX9RnR2ATxoZYmcFQ7QbU57MSJeJr5NjlAoeZWERBRzDoLMw", + "JFKOMyg232ur1a++Z71LhwaE14+5QKsdGk0ejNWptuS6eb1/I9eq1KVm2Oal3u6vXuo78YJ0giyNVssw", + "81I7CbYWxsyKuFKPT6ZK9pomWJrMxxYS/W2ZgotGLtSpQ6FWibLUFU7VPLXISdJTLhVfD73AEMcGKcHT", + "CDKVjG3oC5WFFF0tUrf3n/ndhvi6CablV0JSbask3CI9oASzuXdgdXx4tNF3hXMlgua7BqPOUqs6uKcr", + "NbHGpWoPtV/zapust3JlgxyW5XZx9u2XK0vk3IUf7iGVtDf2ckFWGcOhNeSANq5/E9Zr6ubHtfN674ls", + "p5O9s5ZxEyxUPbS37GY+7P7duJXRsHew9dNf/8/ux7/8yV99q2I3SyK6ERnDfeclmXdNWXFto/eqoMQA", + "WaWVaVuzShGcgNMovCTGSZXg6/J4n/RzoTH/DScLU4CL4oSy/N8rJ/TXPzVfs5bI+B7k5EqW/WoIs7vA", + "h1bcHUcbCRETV4TBRQdu9gYMAg4vyVyiEgilVWlKsFPuk3Jp8AtbAQcK2o8oYPnKAdNWLQ5DkmprwoLx", + "URiL4CB96gXyLRimi+bXcgkDArDNrqoh4Hz2lqE/CBi56poeoq7mvb0n+8BHlJUpub2wxL5FNwn6TTV0", + "NZU9XqPXVEL2iQuyLr2MNkiSqrmDenZhsJvrAQYc5g16b75vGQOv//w2IH/fL8X4/Q4rOJfxHNyAViI5", + "LKx/I7CmP47uuA7MZPakrUpZBRKqWYdSdZvD7BKtzgwhWnQxJA5wtSAS1YLaTrI6uv9WwtSWhdD2Zb9E", + "UHR+aexxscscuAHAZrUIqV2qUJdmVhpJ89qcOr2xptkuIdCZJs3VlAhSWgj4oMABXpNkNi60RU6VAbpN", + "iejWy5aa4imCQqBpbvI7EuSxw4t+0OWARqf4Ou8BfOhYLtw0wTwKwMbtVz9DQaS3rnwlHbsmDMhhVav3", + "oxNVuWgZTRxXLS5GmasW523e9248K6uWSL+mvVVjzqKPCmv6+PFvmKqXXIAd0JzBdOcgR2BjRERACncd", + "wqgV/g9NSDTkmVq+/22lCJu+FDlVvgDRdjYPBia2Ve1XyAKXY1OM4aNPb5AkzARVc20kW5V0RLAg4jAz", + "Gx4ICR3Bz0XHgEn95Qt4C8eegMVXhBFBQ3R4dgL7McEMVGX04RTFdEzCeRgTCym8ALcCSt6boxNrZzqA", + "P7AbqALWc4XmD89OoG61MLZW0O/t9PqwmVPCcEqDg2C3tw1VvDXDwRS3oIQF/GlzEXJ75SSyetDP5hX9", + "lcAJUUTI4OB3T0y/IsKUxJCgdeJJyW5IMRXWcEhjyDQwrEL1t4B25Y7SA3MedwzBW7vJpJrbuEuSvrHL", + "+lFzgtk1MMWdft+YpEzZgxcX9Yy3/mHzM4t+W+lzQB4P9NOCXu90SkvyL51gr7+91nhWliD2dfue4UxN", + "uaCfCAzzyZpEuFGnJ8wEgyMDK2LDXcr7DFiovMN+/6jXS2ZJgsXckaugVcplkzJMtO3OyJWtx/IPPuoh", + "ewkAGMZyyrNYSxNkIt2dua+w6E0+ISzCKZ2RAbPntCknjQUY8wnS57MxW6pbw3RtVj/PQfyZR/MadfPm", + "tnRzXef6LQhcR9WUZAjgYMOmYkuF05cyBhV9JbGYq3nVkcWYGijBLkPurT1PGGaqqOhtaq9fkrn1K3sb", + "bIXiowUeLAsRAK7h/CQ7m/70FUCH9Wd+HefPkCVvVZ1gcBkTxllU6FwuohqLEY5jL8zDJOYjHNsS9ZfE", + "o6K+gjcsUcpAuk65YTwiBiM0naspZ+bvbJQxlZm/R4JfSSK0CmQh5C2tbX1my7pXAAuWAIy7KVCj+9wy", + "Q9z6fEnmX3oDdhglrviQNJ/gWHJbu9+gJVGJXAiu4V0/fG9DdMdRJhVPLEuxcjVZM0yeqTRT9mZbEmVx", + "7+F1qEQtpyQaMMXRZ0EmVCox/7L1uejxS7nqcekVM6WtzzT60jRqOcR69kN41WP9ESDAINCnyyDQf08E", + "1rZLJqfgypDgvpiUl3QjT73XeuFmncIhZijlqYEtAKaaYs1ylTYAjhvHMVKwldy3WtuElWyYj81E8hXE", + "tGlIJm+kto2gNGZpM/X3nvn3kyShID4Hx3+fv/kNwVGl18C8VriNzNUy06coijLQ5KH33oC9wOEUGb0J", + "oOkGAY0GQW5dRJsw1kzaOOluF1Tcn/TQfjLddGj0U6+nmzLa8wH6/bNp5UDvpTQZKn5J2CD40kGlBxOq", + "ptkof/bRT9CmbI7ziiBAG0b2b7oKUIAqURyD5tzALELcytp4jjAqJFDZjzKiDIul5as8pLcU1KY8nsgy", + "MT4PwIM6CA4Gzoc6CDqDgLAZ/GYdrYPgi58CVoluxkEzFbycrp0z0X6/v7k6zdLS16NCV17U2+/Lgva1", + "c2uKh1W6FhUPM7m83jez5dkio27dg+bzM45cdY8fKt4KFc96LkrKG3xfPgcM+8bEGLg1DUzbs7HTwJZa", + "J4YtAMUULA6XFG0MDuo0uIJ5y+ZH3ZxfNCv2mnZZCEOMHf/t3QP/Qb95XQzT7/P76hfHAEmaV/d+XOwI", + "i+UYseO3iF8R9S1wXP++RKnFT31I/n0s/POKWL2vIFpNmm2Rmbtv8kM/QMqHtK2Yl7Wteg5j6p4TptAL", + "+LVn/9dZPABkfBHzycUBMiSM+QTFlNnbuNJtkT4ULS3hI5P1kX9nk0Ac7taGOT//9b//hEFRNvnX//5T", + "a9PmL9juWwYMBXB6L6YECzUiWF0coF8JSbumtIadDCBpkhkRc7TbBzUzFfDIUzFWDtiAvYVCGqVbSwOB", + "JW2DYHowmA9lGZE2a0a/SMcWn8NV0Vkw4d1eNqS81x3dWQwxNjMoTUCfio4HIOGaGrBia38Ffu+ZmXPF", + "f1b3lS94TFfLF0WuleHerhngmgIGSOzbd/DAThptnJ+/2OwhsDEMVwAGC2jMRTNWee79kEmrZZKRKFWB", + "AlQ2ssmkki33/x7bd9o5gG2L35MH2MJyreECNi4PIkjk6PXDVmjjDvbTzbmGff7ZY5cr2eygvfl8y124", + "aKJWhvDtrbPjvUWamyclkj2ECYw2XEy6K9x5dnTiSgFtPhjT38upoWdqC2jkRwfiplzovZllR5yNYxoq", + "1HVjgRoOCclNtSqDPBZx8NaOGmE3rzraYfl826qA9zSedDmOT3Hk3f3pUet0nWOkQGQseO3HSbKKdY6p", + "DLn+tsQt3RCntuqoUV/yfVrmolUOKRPinh85S9UlK55Pjt2GvD/XlO06Y/Wz4R6E4nFNID6gIKzV3Cth", + "mD4mbn6fr6LDhVjiufq2WLN/f1rQfXuxfGz+mNxYUY1sWgoaJOfGA/QVUb+YN+5woW0PnomfE+F2tYOc", + "hlnn0zKfonBKwkszIbiQXm77nphX2pm+pr3vyfIF8qyjsViS/1BRWhi7Ba2WGbgnto7g3dm30MNa5u3t", + "3fNaBvMQGYJNRs5jbUr0YTln4eZ3ddV7L6eZIfajPMzOsjh2Nx4zIhTK65OXz4CtzxCWtFq3d7tt6XHw", + "/u3rLmEhhzi0PIbKr0TZJ7es4ZsFM1P5wSZtbEKTnEzdedak4XzF+ptwQZTXv/+PnZe2Av5/7Lw0NfD/", + "Y/fQVMHfvDNm6d+XaL5vjfsRM59WuGmVaCCaGJSGXaWh5m+1VFLd+9+VnmomvZammtP1h7LaRlktk2up", + "vmqX4k41VtPHA13J5MzmozY8cvGJ35mmer9ePsuRDq6Zyuq1h63HwwX4eeERZSiT5BEGUNKc48rHRkt3", + "dbEhlx4fjnVPjjtAyI4mHWAn2QSRe3Jeu3Hcu3Jr+71/z/VhMqKTjGeynHuSYBVOibTJSjGpCuDHpnYX", + "x3Oj4v0Nc2n/Po+Oe9erf/D9HWn89QU1wtvcQK3S+d1bbXV++77W+U0Ktc1dswBPHQf+t9kQVOiSqNuy", + "cSXXfDHY0Tcuny2C3mtDpTAXEFgQBwP2X9r++F0RnHz8ySXJZP3+zj78Ttjs408uT4adOlYhTAlqsVoP", + "fzuGa78JZJ8DnGuRklcfh6n/AKznAGz+7Qyk4uazvYXkuPCHhdTKQiqRa7mFZNfibk2kKgjWvdtIjt98", + "BLcgJj+spPuwkmQ2HtOQEqaK8mgLQWK2uuIjzC1j9n6oFNxROWhbW0n5plyhgBbg/Pce2HNSoBHet3Hk", + "6gA8zhh5nlpgbWuOFIdhsz3yrfFD/36F8/3bIY+ZxYzCv0i6VOuUvqqbgPSYZAqCEguEEIj6RMJo7XmL", + "PVQUu5RZmnKhpEGLBAXY4MlPtQLsQ5asgkX60CEBA5gS2RkwqBegH5tc/q1LMjdYkJSzHPYxn6nFf/Tl", + "XlWxOB90G92+juUHGm2lY93zNrbQ0Q+nYz2Y6LgXTeukgsi/kW8MMChHJN/JPE/uo58om2w+qghUI6zy", + "uZXwjDyq1hbU27Poulsyr+vbdNCWAHZtNcp/wxN3cZI+rd1h0ZYIiCKKJ4xLRUOXuFvHLP9xQrc+oZdT", + "1svNY1tN1W/Qv+Tisu0R56nu9QhOuvIMv0Ffgh4eoIE9vEsBjG1zGmimufdTcKFk20OmYND6uRjGWaQP", + "QncgOlVyLHgytD8avFq9KywaKLgoQtvqQwsb3fs9OIx+4wrRJI2J1uJJhLqGm/RqWtXfgb5TWSpwuJ4w", + "1NumnBBjwOikKxBkRSRcrrkF24B79sXl8krNmE9Wg2DknTvEBw8KxoAZUHriEOwvUC5koYQWiUmo0NWU", + "hlNAxICyWlBXFcAqcJpe5BBYmwfoFezUMhIYdL4hidCGUMiZ5DExQBezJLk4WERs/XB6Ch8ZMAyDzXpx", + "gBxKa35ASP1WGeEirzz0m8Xt2NCcJHgcmxW90FZjaX6bFvuigCgbMB8OBiNXtkE6RhclSIyLBkwMJ1Bf", + "88mDaVudZmBJMxfFkQDCGd4kLAqaLmJo7EfD2O57q7K0ROYww7hjYI6FwbzmkxzUssLKOE3bsq8dJnDx", + "LEmW8DDaKJVIlSrimfqrVBERAj623N3E3GgDh+YfCl9qRrXlffIis8B+3utGgzLnJZUWqqVaNuZfsyQJ", + "OoEdTwmdbg3tfQXCSb3BxWsxvTIlGJMfevc6ACVVYV9CKKmdHLYIf7PK/da88N37Zy2hou/By1K9zypG", + "QVlRhklAHXVXC+tRIR3AQi7oYqY8kW+PuFl2ZamEZ7vrrYXin9+A0brq1iuv5JiXmbzv66/FETzmJBi5", + "MJsxF/X0+FX3Yt88I93ekixMtQ2H/ODN9d1zrRgzzZZU9ISCpBL8fFDlEnCdwynnssT2IzLFM8qFRWC3", + "XtecM8FlYaxHGz13oVn1wvpvL6x6fmB9TQiXH9k+evC5jbnzf+EeFV+8LFnbucTvOJUaUCAlwmgkKBmj", + "FGeSaG0pSwgyFUYskDfB4dTV7O4N2LspQbZKZcmBkBc1phJdbCcXHTTKFIqxmIC1Yx6aSDpBQp4khEWm", + "8uyATQmeUW2qCRRjRVg470oClYhnpChgok13e0NpCl7ntU47yJXIBQfDRakA7gVKBQEmMuYyq1SbHTCR", + "sf80yJW62Qs30AtEpMKjmMppXisixBFhoRcW8vzbFmO378Q9J2qxRuyD3FneSJY+5CVm2ZeZV+n+Ju43", + "H1mgFheuvmULMb9E6ZXNpmE18vG8qIv7b7ilzVzdHB/oZiYn8bJd/G1cyVQK4/+4llF2S0aZ6Y5Ui8d/", + "t3ctRTXnjFWuW6xP9qYXLnklhJzMa8m8rc/uz5Mb+Mi+EUnYaTTsmzC3i0l/CyLXUvVGMveBnIPWl1Ty", + "ij2gCLaDejj1iYuSlPsmxLDZcLk0LsscJTDYVJz9EMZ1YWzDA24qjJ3HdeECvCSeKeumMW6Sy0XteL8A", + "tg6Bf9Po19rsSoLwwQVfcSNwb8LuJBdvRuCleB5z/L3fy4RcCJPQacsRPx5AsZIvsHTBtAEet04uITou", + "m+TD6elmk5QQaqmMEOoRS4hqWdMw8VRrfDMjQtDIlY48Oj220atUIpGxHnqTUKjneElICoViKM8kgszc", + "np6fS21dLIJXyWHtBIQpMU85ZWrlKIpX72YwX25UOu+e5aSFVPzuL4/BC//4hBTIDq2u2AkstyIVVo3B", + "eC44jTJT71JrW3jEM926liyu0O4EzrYxjYmcS0USE5k3zmLYRAC6a2sy2e9MRmkHUSWR3g8dyMBLiUio", + "lJQzOWC2/HtKhO5bfw7Ff4sgI6/zXuFcap4Z0fdtBLDpwZiYLayaqAbQAlAHNDgItnCabkG5aH+QlB3e", + "VwzpJUSkITlPRjymIYopu5RoI6aXxuhAM4li/cfm0pC2IXx32xWnbr6zNKVP2Jh7i3IYns2Z+ftIQqqK", + "NXeJ+OjE2itS3ixO/sBC+8WaXCnXBMFxV9GE5MnvKFM0pp+MqNONUKloaPJqitRLKMJssy8H7JQood/B", + "gqCQxzEJlXOubKWCh1uDrN/fDVMKKCW7BAYHAq/5cQI9Hp29h/dMoejOgOl/QMPvDs/MTewYWx9BaaCM", + "qCsuLtHJ1psVQb7nQKZ/4yg5M8GlOZDeBf9xfbd+ZnPjHpINW5Snywwgnn73YZxWg/vhLXic3gKAlshn", + "szEROASlWE4zFfEr5vcMzHicJfof5o+TVQAlCofTD/DqN6PtmuGs7MZN8FFsSjuniJiiQQ9yQWEI9ljj", + "SzXh3BRAialE7nlPgUP1PXL37Tvly3T8Bq8mLUVdQa5vZm/d98lnx+Bwt8r0eCzb3HCam4niy71PV5g2", + "e59+jnl4KVHGFI0roAbabgMcUP1jgdtoL/5ATYDsSFdKHJHrlApAsKnBIyCiZywRRoqIhDIcb8GcTSOA", + "QOm8WHjGKSQphzGFNDEaEZTyOAaUnaspYUjPBhxVroHSPa20FSDK75SvGBVHIxLyhDhUzk2f6fY3TNVL", + "LqoQm9+KXHxXor+ej56qnucKVNHmHr8KZfQUX0NYc5TZa2I3oo1XvPjRuII6CNZmEOz25SDooEGwkwwC", + "vQJHGFyoWKEnKKEsU0T20LHxb0Ea6n4fSRJyFkkHDuo8eLt92ZSUatiyIcNxH767T7XHchWQ8q3txCce", + "9HtIfw8JNmijvOHsnow6sOkixDMFAdxuX9m3IqLAPbJ57zewpT3yw7ZvI8n/ZrdvRUbBKmtxWVp6I9lz", + "+MiVXjeXVDHlskCdRCFOcUjVvINwHPOw8B5kMr8d6OZDGQmCL7UN1RuwtzlwpU2EQEdn7zvOaYYiKi9N", + "C9Yv1kNvZkTIbJQPDoE0MB48WAwSDZjiKMRxmMWab8l4TELIYYhpQpVs8KvlQ7nLMohFJ56Fdw9z2JrH", + "5Uzy8wSsXsEWssZxW2aptwQJY0yTslOpThxQfeFKF9y+I90o18fwOLbXW6HgUiLbVJfEdEJHsb2skT30", + "TqscOCEDlsaYMSJQJk3ckR56NxVEyswkxugGoM6s4agOKoBOUsGVdRPHnAtpPLuawz+cIqlIuoTN3pqW", + "T2HOdwQTbBq3PT2QwVAbQ/OxZF9BekEMpxiCaz7Sx/QDBPuYAT00nPBj2fjvBJ1MiNC7Ahsha65GzbZ2", + "5DSbvpLp0YiRf56/1Q4jP2+1FM1dinReClQxdC8OQYFe5wbW0/klbcQysY/Wy774VX/Usu9qlL9/EPbR", + "V87yeyk9dl4Krm6LrF9w+GMDuS+NvLJVKwkKq+EIWmck3GWGQGvcgQeDG3jMKAO4knbQBCfw7TFC/36z", + "4+4bZvtx81YFJaBSWKchVWo1fOc3wYF3g9v5wNmhN8Dt/KbylQB38eHyRr+pTKWKH9AVD/nukTnvKkHJ", + "wHMCjEVTgpKRejaQYKmh9MG+085Msi1+Txq8vXteQ393ZP9h9bcwGUrE8rvsTG60w20hSarm7nKRj2sX", + "gJJ+gmQMH/BDHkNwd3gLN7hevz32cHzaeLn+o57Wvd3fF0WHT44ffxGt8p6rHCxb+tTpYhFO6Yw0O92r", + "O9iSKBWkm/IULlciQzBLD3eWKSx6k0/INm+xquy/EHUQxyRCERUkVPEcUaY4SATTx58lElxbAvCci7nP", + "mV7euS8FTw7tbFach3ZPWWdYceebzLsRVrg7c9JmiQvtK27a3d22FniIMvTqZ7RBrpUwiLtorC0fRMc5", + "Scl1SEgkgSc3ywPe7jd4NuknMpyM2oxyCXbyG4tNjcJMKp64tT85RhtQbGFCmF4LreqPQZNNBZ/RyBQi", + "LYg647Gh6nYDQdf1u2qlIq+U4YwLM7gH0WHaHEiTTzStigUTuhAcBCPKMAxuJUpxdU+ZhCrdH6aQ1lDs", + "Hcc5wY8jzFp+G87Y0ZyojRxHRMW5gcbb/HHMPeZjrhyY6s60ymnXrlRku1jVliGkdwGYm8cx36/b+sO3", + "E15J5aOMrLSu81lukDa5zb8tFuzf3/lw3+7yD484HP8VccZ3yVUODegWfQzzmoc4RhGZkZinUEXSvBt0", + "gkzEwUEwVSo92NqK9XtTLtXBs/6zfvDl45f/PwAA//92jJRrQ3gBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index adc070c4..8fe78603 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -260,6 +260,46 @@ func (p *Paths) SnapshotGuestDir(snapshotID string) string { return filepath.Join(p.SnapshotDir(snapshotID), "guest") } +// Template path methods + +// TemplatesDir returns the root directory for VM templates. +// A template is a tagged Standby instance promoted to a "fork-only" parent +// whose snapshot can be reused for many forked instances. +func (p *Paths) TemplatesDir() string { + return filepath.Join(p.dataDir, "templates") +} + +// TemplateDir returns the directory for a specific template's metadata. +func (p *Paths) TemplateDir(id string) string { + return filepath.Join(p.TemplatesDir(), id) +} + +// TemplateMetadata returns the path to a template's metadata.json file. +func (p *Paths) TemplateMetadata(id string) string { + return filepath.Join(p.TemplateDir(id), "template.json") +} + +// TemplateUffdDir returns the directory that holds a template's per-fork +// userfaultfd page-server sockets. Sockets cannot live under the data dir +// because Unix domain socket paths are limited to 108 bytes (sun_path); a +// deep DataDir + the cuid2 template+fork ids easily blow that. Anchoring +// at a short, fixed runtime root keeps the absolute path well under the +// limit (~30 chars for typical ids) regardless of where the data dir lives. +// +// Sockets are ephemeral — losing them on reboot is fine; firecracker +// reconnects on restore. +func (p *Paths) TemplateUffdDir(id string) string { + return filepath.Join(uffdRuntimeRoot(), id) +} + +// uffdRuntimeRoot returns the short root used to hold uffd sockets. It is +// not configurable on purpose: the only requirement is that it stay short +// and writable, and /tmp meets both. Tests get isolation from the cuid +// segment in TemplateUffdDir. +func uffdRuntimeRoot() string { + return filepath.Join("/tmp", "h-uffd") +} + // Device path methods // DevicesDir returns the root devices directory. diff --git a/lib/templates/registry.go b/lib/templates/registry.go new file mode 100644 index 00000000..fed3f169 --- /dev/null +++ b/lib/templates/registry.go @@ -0,0 +1,252 @@ +package templates + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// Registry persists and indexes templates. The default file-backed +// implementation stores one JSON file per template under +// paths.TemplatesDir(); higher-level callers (the instances manager) hold +// the registry and read it as a stable index. +// +// Registry is concurrency-safe; in-process locking keeps reads and writes +// consistent. Cross-process callers should not be writing to the same data +// dir simultaneously today; if/when that changes we'd add file locking. +type Registry interface { + // Save inserts or replaces a template record. + Save(ctx context.Context, t *Template) error + + // Get returns a template by its ID. ErrNotFound when missing. + Get(ctx context.Context, id string) (*Template, error) + + // GetByName resolves a template by its unique name. + GetByName(ctx context.Context, name string) (*Template, error) + + // List returns all templates, optionally filtered. + List(ctx context.Context, filter *ListFilter) ([]*Template, error) + + // Delete removes a template. Returns ErrInUse when ForkCount > 0. + Delete(ctx context.Context, id string) error + + // IncrementForkCount atomically bumps the fork refcount on a + // template. Used at fork creation time. + IncrementForkCount(ctx context.Context, id string) (*Template, error) + + // DecrementForkCount atomically drops the fork refcount on a + // template (floor 0). Used when a fork is deleted. Touching + // templates that were already deleted is a no-op. + DecrementForkCount(ctx context.Context, id string) (*Template, error) +} + +// ListFilter narrows the templates returned by Registry.List. +type ListFilter struct { + // HypervisorType, when non-empty, restricts results to templates that + // share the given hypervisor type. Forks must match the hypervisor + // of their template. + HypervisorType hypervisor.Type + + // ImageDigest, when non-empty, restricts results to templates whose + // resolved image digest equals the given value. Useful when picking + // a fan-out parent for a particular image revision. + ImageDigest string +} + +// FileRegistry is the default Registry implementation. It stores each +// template as a JSON file under TemplatesDir//template.json. +type FileRegistry struct { + dir string + mu sync.Mutex +} + +// NewFileRegistry returns a Registry that persists to dir. The directory is +// created on first write. +func NewFileRegistry(dir string) *FileRegistry { + return &FileRegistry{dir: dir} +} + +func (r *FileRegistry) path(id string) string { + return filepath.Join(r.dir, id, "template.json") +} + +func (r *FileRegistry) ensureDir(id string) error { + return os.MkdirAll(filepath.Join(r.dir, id), 0o755) +} + +func (r *FileRegistry) writeLocked(t *Template) error { + if err := t.Validate(); err != nil { + return fmt.Errorf("%w: %v", ErrInvalid, err) + } + if err := r.ensureDir(t.ID); err != nil { + return fmt.Errorf("create template dir: %w", err) + } + data, err := json.MarshalIndent(t, "", " ") + if err != nil { + return fmt.Errorf("marshal template: %w", err) + } + tmp := r.path(t.ID) + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("write template tmp: %w", err) + } + if err := os.Rename(tmp, r.path(t.ID)); err != nil { + return fmt.Errorf("rename template tmp: %w", err) + } + return nil +} + +func (r *FileRegistry) readLocked(id string) (*Template, error) { + data, err := os.ReadFile(r.path(id)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%w: id=%s", ErrNotFound, id) + } + return nil, fmt.Errorf("read template: %w", err) + } + var t Template + if err := json.Unmarshal(data, &t); err != nil { + return nil, fmt.Errorf("unmarshal template %s: %w", id, err) + } + return &t, nil +} + +func (r *FileRegistry) Save(ctx context.Context, t *Template) error { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + return r.writeLocked(t) +} + +func (r *FileRegistry) Get(ctx context.Context, id string) (*Template, error) { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + return r.readLocked(id) +} + +func (r *FileRegistry) GetByName(ctx context.Context, name string) (*Template, error) { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + all, err := r.listLocked() + if err != nil { + return nil, err + } + for _, t := range all { + if t.Name == name { + return t, nil + } + } + return nil, fmt.Errorf("%w: name=%s", ErrNotFound, name) +} + +func (r *FileRegistry) List(ctx context.Context, filter *ListFilter) ([]*Template, error) { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + all, err := r.listLocked() + if err != nil { + return nil, err + } + if filter == nil { + return all, nil + } + out := make([]*Template, 0, len(all)) + for _, t := range all { + if filter.HypervisorType != "" && t.HypervisorType != filter.HypervisorType { + continue + } + if filter.ImageDigest != "" && t.ImageDigest != filter.ImageDigest { + continue + } + out = append(out, t) + } + return out, nil +} + +func (r *FileRegistry) listLocked() ([]*Template, error) { + entries, err := os.ReadDir(r.dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read templates dir: %w", err) + } + out := make([]*Template, 0, len(entries)) + for _, e := range entries { + if !e.IsDir() { + continue + } + t, err := r.readLocked(e.Name()) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue + } + return nil, err + } + out = append(out, t) + } + sort.Slice(out, func(i, j int) bool { + return out[i].CreatedAt.Before(out[j].CreatedAt) + }) + return out, nil +} + +func (r *FileRegistry) Delete(ctx context.Context, id string) error { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + t, err := r.readLocked(id) + if err != nil { + return err + } + if t.ForkCount > 0 { + return fmt.Errorf("%w: %d live forks reference template %s", ErrInUse, t.ForkCount, id) + } + if err := os.RemoveAll(filepath.Join(r.dir, id)); err != nil { + return fmt.Errorf("remove template dir: %w", err) + } + return nil +} + +func (r *FileRegistry) IncrementForkCount(ctx context.Context, id string) (*Template, error) { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + t, err := r.readLocked(id) + if err != nil { + return nil, err + } + t.ForkCount++ + if err := r.writeLocked(t); err != nil { + return nil, err + } + return t, nil +} + +func (r *FileRegistry) DecrementForkCount(ctx context.Context, id string) (*Template, error) { + _ = ctx + r.mu.Lock() + defer r.mu.Unlock() + t, err := r.readLocked(id) + if err != nil { + if errors.Is(err, ErrNotFound) { + return nil, nil + } + return nil, err + } + if t.ForkCount > 0 { + t.ForkCount-- + } + if err := r.writeLocked(t); err != nil { + return nil, err + } + return t, nil +} diff --git a/lib/templates/registry_test.go b/lib/templates/registry_test.go new file mode 100644 index 00000000..79a93031 --- /dev/null +++ b/lib/templates/registry_test.go @@ -0,0 +1,125 @@ +package templates + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestRegistry(t *testing.T) *FileRegistry { + t.Helper() + return NewFileRegistry(filepath.Join(t.TempDir(), "templates")) +} + +func sampleTemplate(id, name string) *Template { + return &Template{ + ID: id, + Name: name, + SourceInstanceID: "src-" + id, + Image: "docker.io/library/alpine:latest", + ImageDigest: "sha256:deadbeef", + HypervisorType: hypervisor.TypeFirecracker, + HypervisorVersion: "v1.14.2", + MemoryBytes: 1 << 30, + VCPUs: 2, + CreatedAt: time.Now().UTC(), + } +} + +func TestFileRegistry_SaveGet(t *testing.T) { + r := newTestRegistry(t) + tpl := sampleTemplate("t1", "alpine-warm") + + require.NoError(t, r.Save(context.Background(), tpl)) + + got, err := r.Get(context.Background(), "t1") + require.NoError(t, err) + assert.Equal(t, "alpine-warm", got.Name) + assert.Equal(t, hypervisor.TypeFirecracker, got.HypervisorType) +} + +func TestFileRegistry_GetByName(t *testing.T) { + r := newTestRegistry(t) + require.NoError(t, r.Save(context.Background(), sampleTemplate("t1", "alpha"))) + require.NoError(t, r.Save(context.Background(), sampleTemplate("t2", "beta"))) + + got, err := r.GetByName(context.Background(), "beta") + require.NoError(t, err) + assert.Equal(t, "t2", got.ID) + + _, err = r.GetByName(context.Background(), "missing") + assert.True(t, errors.Is(err, ErrNotFound)) +} + +func TestFileRegistry_List_Filter(t *testing.T) { + r := newTestRegistry(t) + a := sampleTemplate("a", "a") + b := sampleTemplate("b", "b") + b.ImageDigest = "sha256:other" + c := sampleTemplate("c", "c") + c.HypervisorType = hypervisor.TypeCloudHypervisor + + require.NoError(t, r.Save(context.Background(), a)) + require.NoError(t, r.Save(context.Background(), b)) + require.NoError(t, r.Save(context.Background(), c)) + + all, err := r.List(context.Background(), nil) + require.NoError(t, err) + assert.Len(t, all, 3) + + byHV, err := r.List(context.Background(), &ListFilter{HypervisorType: hypervisor.TypeFirecracker}) + require.NoError(t, err) + assert.Len(t, byHV, 2) + + byDigest, err := r.List(context.Background(), &ListFilter{ImageDigest: "sha256:deadbeef"}) + require.NoError(t, err) + assert.Len(t, byDigest, 2) +} + +func TestFileRegistry_Refcount(t *testing.T) { + r := newTestRegistry(t) + require.NoError(t, r.Save(context.Background(), sampleTemplate("t1", "a"))) + + got, err := r.IncrementForkCount(context.Background(), "t1") + require.NoError(t, err) + assert.Equal(t, 1, got.ForkCount) + + got, err = r.IncrementForkCount(context.Background(), "t1") + require.NoError(t, err) + assert.Equal(t, 2, got.ForkCount) + + err = r.Delete(context.Background(), "t1") + assert.True(t, errors.Is(err, ErrInUse)) + + got, err = r.DecrementForkCount(context.Background(), "t1") + require.NoError(t, err) + assert.Equal(t, 1, got.ForkCount) + got, err = r.DecrementForkCount(context.Background(), "t1") + require.NoError(t, err) + assert.Equal(t, 0, got.ForkCount) + + err = r.Delete(context.Background(), "t1") + require.NoError(t, err) + + _, err = r.Get(context.Background(), "t1") + assert.True(t, errors.Is(err, ErrNotFound)) +} + +func TestFileRegistry_DecrementMissingIsNoop(t *testing.T) { + r := newTestRegistry(t) + got, err := r.DecrementForkCount(context.Background(), "missing") + require.NoError(t, err) + assert.Nil(t, got) +} + +func TestFileRegistry_SaveValidates(t *testing.T) { + r := newTestRegistry(t) + err := r.Save(context.Background(), &Template{Name: "x"}) + assert.True(t, errors.Is(err, ErrInvalid)) +} diff --git a/lib/templates/template.go b/lib/templates/template.go new file mode 100644 index 00000000..93968d90 --- /dev/null +++ b/lib/templates/template.go @@ -0,0 +1,108 @@ +// Package templates models VM templates: tagged Standby instances promoted +// to "fork-only" parents whose snapshot can be reused for many forked +// instances. Templates are the foundation for one-snapshot-to-N-forks +// fan-out: rather than every fork copying or diffing against its own private +// snapshot, forks descend from a shared template. The actual sharing of +// memory and rootfs CoW between fork and template is implemented in the +// hypervisor and forkvm packages; this package just owns the lifecycle and +// indexing primitives. +package templates + +import ( + "errors" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/tags" +) + +// Common errors returned by the templates package. +var ( + ErrNotFound = errors.New("template not found") + ErrAlreadyExists = errors.New("template already exists") + ErrInUse = errors.New("template is in use by one or more forks") + ErrInvalid = errors.New("invalid template") +) + +// Template is the persisted record describing a fork-only parent instance. +// It points at a source instance directory whose snapshot artifacts are +// shared by many forks, and tracks how many live forks reference it so we +// don't GC the underlying memory file or rootfs out from under them. +type Template struct { + // ID is the template's stable identifier. It is independent of the + // source instance ID so a template can outlive its source. + ID string `json:"id"` + + // Name is a human-readable label, unique across templates. + Name string `json:"name"` + + // SourceInstanceID is the instance the template was promoted from. + // Its on-disk directory holds the canonical snapshot used by forks. + SourceInstanceID string `json:"source_instance_id"` + + // Image is the OCI reference the source instance was created from. + // Used for indexing templates by image when picking a fanout parent. + Image string `json:"image,omitempty"` + + // ImageDigest is the resolved image digest (sha256:…) at the time of + // promotion. Two templates with the same digest are interchangeable + // for the purposes of fan-out pool selection. + ImageDigest string `json:"image_digest,omitempty"` + + // HypervisorType records which hypervisor produced the snapshot. + // Templates can only be forked by the same hypervisor type. + HypervisorType hypervisor.Type `json:"hypervisor_type"` + + // HypervisorVersion is the hypervisor binary version used to take the + // snapshot. Restoring on a different version may work but isn't + // guaranteed; we store it so we can warn or refuse on mismatch. + HypervisorVersion string `json:"hypervisor_version,omitempty"` + + // MemoryBytes is the guest memory size the snapshot was taken at. + // Forks must be configured with at least this much memory. + MemoryBytes int64 `json:"memory_bytes,omitempty"` + + // VCPUs is the vCPU count the snapshot was taken at. Snapshots are + // vCPU-count-specific on most hypervisors. + VCPUs int `json:"vcpus,omitempty"` + + // Tags carries arbitrary user metadata, e.g. release identifiers. + Tags tags.Tags `json:"tags,omitempty"` + + // CreatedAt is when the template was first registered. + CreatedAt time.Time `json:"created_at"` + + // LastUsedAt is updated whenever a fork is created from the template. + // Useful as a proxy for popularity when GC-ing stale templates. + LastUsedAt time.Time `json:"last_used_at,omitempty"` + + // ForkCount is the number of live forks descended from this template. + // While > 0, the template (and its underlying snapshot files) must not + // be deleted. PR 4 owns reference counting; PR 2 just records the field. + ForkCount int `json:"fork_count"` + + // HotPagesPath optionally points at a baked "hot page list" used by + // the UFFD page server to prefetch known-touched pages before resume. + // PR 8 wires this in; PR 2 just reserves the field. + HotPagesPath string `json:"hot_pages_path,omitempty"` +} + +// Validate checks that required fields are populated. +func (t *Template) Validate() error { + if t == nil { + return errors.New("nil template") + } + if t.ID == "" { + return errors.New("template id is required") + } + if t.Name == "" { + return errors.New("template name is required") + } + if t.SourceInstanceID == "" { + return errors.New("template source_instance_id is required") + } + if t.HypervisorType == "" { + return errors.New("template hypervisor_type is required") + } + return nil +} diff --git a/lib/uffd/hotpages.go b/lib/uffd/hotpages.go new file mode 100644 index 00000000..25eecad9 --- /dev/null +++ b/lib/uffd/hotpages.go @@ -0,0 +1,176 @@ +package uffd + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" +) + +// HotPage points at a single page-aligned location inside a registered +// memory region. Region is the index into the handshake's mappings list; +// PageOffset is the byte offset of the page within that region (always +// a multiple of the server's page size). +type HotPage struct { + Region uint32 + PageOffset uint64 +} + +// HotPageList is the persisted "what pages should we eagerly populate +// before the guest unpauses" list. PR 8 records one of these during a +// template's first fork warm-up and bakes it into Template.HotPagesPath; +// later forks call Server.Prefetch with the loaded list to skip the +// fault round-trips on those pages. +// +// Concurrent Add/Snapshot is safe; Save and Load are not — callers +// generally Save once at the end of warmup and Load once at boot. +type HotPageList struct { + mu sync.Mutex + pages []HotPage +} + +// hotPagesFileMagic prefixes saved files so we can refuse to load +// arbitrary garbage. The version byte exists so a future format change +// can be rejected loudly instead of silently misinterpreted. +var hotPagesFileMagic = []byte("HPL1") + +// Add records a single hot page. Duplicates are tolerated; Snapshot +// dedups before returning. +func (h *HotPageList) Add(p HotPage) { + h.mu.Lock() + h.pages = append(h.pages, p) + h.mu.Unlock() +} + +// Len returns the number of recorded pages (with duplicates). +func (h *HotPageList) Len() int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.pages) +} + +// Snapshot returns a sorted, deduplicated copy of the recorded pages. +// Sort order is (Region, PageOffset) so prefetch issues sequential +// reads against the template mem-file. +func (h *HotPageList) Snapshot() []HotPage { + h.mu.Lock() + src := make([]HotPage, len(h.pages)) + copy(src, h.pages) + h.mu.Unlock() + + sort.Slice(src, func(i, j int) bool { + if src[i].Region != src[j].Region { + return src[i].Region < src[j].Region + } + return src[i].PageOffset < src[j].PageOffset + }) + out := src[:0] + var last HotPage + for i, p := range src { + if i == 0 || p != last { + out = append(out, p) + last = p + } + } + return out +} + +// Save atomically writes the deduplicated snapshot to path. The format +// is: 4-byte magic ("HPL1"), uvarint count, then for each page a +// uvarint region index and a uvarint page offset. Atomic via tmp+rename. +func (h *HotPageList) Save(path string) error { + pages := h.Snapshot() + tmp := path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return fmt.Errorf("uffd: create hot pages tmp: %w", err) + } + bw := bufio.NewWriter(f) + if _, err := bw.Write(hotPagesFileMagic); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("uffd: write hot pages magic: %w", err) + } + var ibuf [binary.MaxVarintLen64]byte + n := binary.PutUvarint(ibuf[:], uint64(len(pages))) + if _, err := bw.Write(ibuf[:n]); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("uffd: write hot pages count: %w", err) + } + for _, p := range pages { + n = binary.PutUvarint(ibuf[:], uint64(p.Region)) + if _, err := bw.Write(ibuf[:n]); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("uffd: write hot pages region: %w", err) + } + n = binary.PutUvarint(ibuf[:], p.PageOffset) + if _, err := bw.Write(ibuf[:n]); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("uffd: write hot pages offset: %w", err) + } + } + if err := bw.Flush(); err != nil { + _ = f.Close() + _ = os.Remove(tmp) + return fmt.Errorf("uffd: flush hot pages: %w", err) + } + if err := f.Close(); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("uffd: close hot pages tmp: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("uffd: rename hot pages: %w", err) + } + return nil +} + +// LoadHotPageList reads a HotPageList from path. Returns an empty list +// (not an error) when path does not exist; the absence of a baked +// hot-page file simply means "don't prefetch." +func LoadHotPageList(path string) (*HotPageList, error) { + clean := filepath.Clean(path) + data, err := os.ReadFile(clean) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return &HotPageList{}, nil + } + return nil, fmt.Errorf("uffd: read hot pages: %w", err) + } + if len(data) < len(hotPagesFileMagic) { + return nil, errors.New("uffd: hot pages file truncated") + } + if string(data[:len(hotPagesFileMagic)]) != string(hotPagesFileMagic) { + return nil, errors.New("uffd: hot pages file has bad magic") + } + rest := data[len(hotPagesFileMagic):] + count, n := binary.Uvarint(rest) + if n <= 0 { + return nil, errors.New("uffd: hot pages file has bad count") + } + rest = rest[n:] + out := &HotPageList{pages: make([]HotPage, 0, count)} + for i := uint64(0); i < count; i++ { + region, n := binary.Uvarint(rest) + if n <= 0 { + return nil, fmt.Errorf("uffd: hot pages file truncated at entry %d (region)", i) + } + rest = rest[n:] + offset, n := binary.Uvarint(rest) + if n <= 0 { + return nil, fmt.Errorf("uffd: hot pages file truncated at entry %d (offset)", i) + } + rest = rest[n:] + out.pages = append(out.pages, HotPage{Region: uint32(region), PageOffset: offset}) + } + if len(rest) != 0 { + return nil, fmt.Errorf("uffd: hot pages file has %d trailing bytes", len(rest)) + } + return out, nil +} diff --git a/lib/uffd/hotpages_test.go b/lib/uffd/hotpages_test.go new file mode 100644 index 00000000..4441194f --- /dev/null +++ b/lib/uffd/hotpages_test.go @@ -0,0 +1,66 @@ +package uffd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHotPageList_SnapshotSortsAndDedups(t *testing.T) { + var l HotPageList + l.Add(HotPage{Region: 1, PageOffset: 8192}) + l.Add(HotPage{Region: 0, PageOffset: 4096}) + l.Add(HotPage{Region: 0, PageOffset: 4096}) // duplicate + l.Add(HotPage{Region: 0, PageOffset: 0}) + + got := l.Snapshot() + want := []HotPage{ + {Region: 0, PageOffset: 0}, + {Region: 0, PageOffset: 4096}, + {Region: 1, PageOffset: 8192}, + } + assert.Equal(t, want, got) +} + +func TestHotPageList_SaveLoadRoundTrip(t *testing.T) { + var l HotPageList + l.Add(HotPage{Region: 0, PageOffset: 0}) + l.Add(HotPage{Region: 0, PageOffset: 4096}) + l.Add(HotPage{Region: 2, PageOffset: 1 << 20}) + + path := filepath.Join(t.TempDir(), "hot.bin") + require.NoError(t, l.Save(path)) + + got, err := LoadHotPageList(path) + require.NoError(t, err) + assert.Equal(t, l.Snapshot(), got.Snapshot()) +} + +func TestLoadHotPageList_MissingReturnsEmpty(t *testing.T) { + got, err := LoadHotPageList(filepath.Join(t.TempDir(), "absent.bin")) + require.NoError(t, err) + assert.Equal(t, 0, got.Len()) +} + +func TestLoadHotPageList_BadMagic(t *testing.T) { + path := filepath.Join(t.TempDir(), "bad.bin") + require.NoError(t, writeFile(path, []byte("XXXX\x00"))) + _, err := LoadHotPageList(path) + assert.Error(t, err) +} + +func TestLoadHotPageList_TruncatedAtEntry(t *testing.T) { + path := filepath.Join(t.TempDir(), "trunc.bin") + // magic + count=2 + only one entry + data := append([]byte("HPL1"), 0x02, 0x00, 0x00) // count=2, region=0, offset=0 + require.NoError(t, writeFile(path, data)) + _, err := LoadHotPageList(path) + assert.Error(t, err) +} + +func writeFile(path string, data []byte) error { + return os.WriteFile(path, data, 0o600) +} diff --git a/lib/uffd/server_linux.go b/lib/uffd/server_linux.go new file mode 100644 index 00000000..618addb4 --- /dev/null +++ b/lib/uffd/server_linux.go @@ -0,0 +1,358 @@ +//go:build linux + +package uffd + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "os" + "sync" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +// userfaultfd ioctl numbers and feature flags. The constants are derived +// from : _IOWR(0xAA, ...) with the size of each +// argument struct in bits 16–29. +const ( + uffdAPI = 0xAA + uffdAPIFeature = 0x0 // we only need missing-page faults; no extra features. + + uffdioAPI = 0xC018AA3F // _IOWR(0xAA, 0x3F, struct uffdio_api{24}) + uffdioRegister = 0xC020AA00 // _IOWR(0xAA, 0x00, struct uffdio_register{32}) + uffdioCopyIoctl = 0xC028AA03 // _IOWR(0xAA, 0x03, struct uffdio_copy{40}) + uffdioZeropage = 0xC020AA04 // _IOWR(0xAA, 0x04, struct uffdio_zeropage{32}) + uffdRegMissing = 1 << 0 + uffdEventPagefnt = 0x12 // UFFD_EVENT_PAGEFAULT +) + +// uffdMsg mirrors struct uffd_msg from . It is a +// 32-byte fixed-size record; we only consume the pagefault arm. +type uffdMsg struct { + Event uint8 + _ uint8 + _ uint16 + _ uint32 + Pagefault struct { + Flags uint64 + Address uint64 + Ptid uint32 + _ uint32 + } +} + +// uffdioAPIArg is struct uffdio_api. +type uffdioAPIArg struct { + API uint64 + Features uint64 + Ioctls uint64 +} + +// uffdioRegisterArg is struct uffdio_register. +type uffdioRegisterArg struct { + Start uint64 + Len uint64 + Mode uint64 + Ioctls uint64 +} + +// uffdioCopyArg is struct uffdio_copy. +type uffdioCopyArg struct { + Dst uint64 + Src uint64 + Len uint64 + Mode uint64 + Copy int64 +} + +// startListener opens the per-fork UDS, accepts firecracker's connection, +// receives the userfaultfd via SCM_RIGHTS plus the JSON handshake, and +// then runs the page-fault loop. The returned closer stops accept, +// signals the handler, and removes the socket file. +func (s *Server) startListener(ctx context.Context, forkID string, socketPath string) (func() error, error) { + // Remove any stale socket file from a prior run; UDS bind fails otherwise. + _ = os.Remove(socketPath) + ln, err := net.Listen("unix", socketPath) + if err != nil { + return nil, fmt.Errorf("uffd: listen %s: %w", socketPath, err) + } + + hctx, hcancel := context.WithCancel(ctx) + + var ( + wg sync.WaitGroup + mu sync.Mutex + uffdFd int = -1 + closed bool + ) + + closer := func() error { + mu.Lock() + if closed { + mu.Unlock() + wg.Wait() + return nil + } + closed = true + fd := uffdFd + uffdFd = -1 + mu.Unlock() + + hcancel() + _ = ln.Close() + if fd >= 0 { + _ = unix.Close(fd) + } + wg.Wait() + _ = os.Remove(socketPath) + return nil + } + + wg.Add(1) + go func() { + defer wg.Done() + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + + fd, regions, err := receiveHandshake(conn) + if err != nil { + return + } + mu.Lock() + if closed { + mu.Unlock() + _ = unix.Close(fd) + return + } + uffdFd = fd + mu.Unlock() + + if err := uffdAPIHandshake(fd); err != nil { + return + } + for _, r := range regions { + if err := uffdRegisterRegion(fd, r); err != nil { + return + } + } + + s.installPrefetcher(forkID, func(list *HotPageList) error { + return s.prefetchInto(fd, regions, list) + }) + + s.servePageFaults(hctx, fd, regions, forkID) + }() + + return closer, nil +} + +// receiveHandshake reads firecracker's JSON payload and the userfaultfd +// over a single recvmsg(2) call. Firecracker sends them together; if the +// kernel splits them across reads we loop until the fd arrives. +func receiveHandshake(conn net.Conn) (int, []MemoryRegion, error) { + uc, ok := conn.(*net.UnixConn) + if !ok { + return -1, nil, errors.New("uffd: connection is not a unix socket") + } + f, err := uc.File() + if err != nil { + return -1, nil, fmt.Errorf("uffd: get fd from unix conn: %w", err) + } + defer f.Close() + + // Read until we have the SCM_RIGHTS fd. The JSON body is small, so + // a 4 KiB buffer plus one OOB control message is plenty. + buf := make([]byte, 4096) + oob := make([]byte, unix.CmsgSpace(4)) + var ( + jsonBytes []byte + fd int = -1 + ) + for fd < 0 { + n, oobn, _, _, err := unix.Recvmsg(int(f.Fd()), buf, oob, 0) + if err != nil { + return -1, nil, fmt.Errorf("uffd: recvmsg: %w", err) + } + if n > 0 { + jsonBytes = append(jsonBytes, buf[:n]...) + } + if oobn > 0 { + scms, perr := unix.ParseSocketControlMessage(oob[:oobn]) + if perr != nil { + return -1, nil, fmt.Errorf("uffd: parse cmsg: %w", perr) + } + for _, scm := range scms { + fds, ferr := unix.ParseUnixRights(&scm) + if ferr != nil { + return -1, nil, fmt.Errorf("uffd: parse fds: %w", ferr) + } + if len(fds) > 0 { + fd = fds[0] + for _, extra := range fds[1:] { + _ = unix.Close(extra) + } + } + } + } + if n == 0 && oobn == 0 { + return -1, nil, io.ErrUnexpectedEOF + } + } + + hs, err := parseHandshake(jsonBytes) + if err != nil { + _ = unix.Close(fd) + return -1, nil, err + } + return fd, hs.Mappings, nil +} + +func uffdAPIHandshake(fd int) error { + api := uffdioAPIArg{API: uffdAPI, Features: uffdAPIFeature} + if err := ioctl(fd, uffdioAPI, unsafe.Pointer(&api)); err != nil { + return fmt.Errorf("uffd: UFFDIO_API: %w", err) + } + return nil +} + +func uffdRegisterRegion(fd int, r MemoryRegion) error { + reg := uffdioRegisterArg{ + Start: uint64(r.BaseHostAddr), + Len: r.Size, + Mode: uffdRegMissing, + } + if err := ioctl(fd, uffdioRegister, unsafe.Pointer(®)); err != nil { + return fmt.Errorf("uffd: UFFDIO_REGISTER: %w", err) + } + return nil +} + +// servePageFaults blocks reading uffd events on fd. For each +// UFFD_EVENT_PAGEFAULT we look up the region containing the faulting +// address, read a page from the template mem-file, and call UFFDIO_COPY +// to satisfy the fault. +func (s *Server) servePageFaults(ctx context.Context, fd int, regions []MemoryRegion, forkID string) { + page := make([]byte, s.pageSize) + var msg uffdMsg + msgSize := int(unsafe.Sizeof(msg)) + rawBuf := make([]byte, msgSize) + + for { + if ctx.Err() != nil { + return + } + n, err := unix.Read(fd, rawBuf) + if err != nil { + if errors.Is(err, syscall.EINTR) { + continue + } + return + } + if n != msgSize { + return + } + event := rawBuf[0] + if event != uffdEventPagefnt { + continue + } + // pagefault.address starts at offset 16 of uffd_msg. + addr := binary.LittleEndian.Uint64(rawBuf[16:24]) + if err := s.copyPageForFault(fd, regions, addr, page); err != nil { + return + } + } +} + +func (s *Server) copyPageForFault(fd int, regions []MemoryRegion, addr uint64, page []byte) error { + pageSize := uint64(s.pageSize) + pageStart := addr &^ (pageSize - 1) + + for idx, r := range regions { + base := uint64(r.BaseHostAddr) + if pageStart < base || pageStart >= base+r.Size { + continue + } + regionOff := pageStart - base + offset := int64(r.MemFileOffset + regionOff) + if _, err := s.memFile.ReadAt(page, offset); err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("uffd: read template at %d: %w", offset, err) + } + copyArg := uffdioCopyArg{ + Dst: pageStart, + Src: uint64(uintptr(unsafe.Pointer(&page[0]))), + Len: pageSize, + } + if err := ioctl(fd, uffdioCopyIoctl, unsafe.Pointer(©Arg)); err != nil { + // Spurious/duplicate faults can race other vCPUs; treat + // them as benign and keep serving. + if errors.Is(err, syscall.EEXIST) || errors.Is(err, syscall.EAGAIN) { + return nil + } + return fmt.Errorf("uffd: UFFDIO_COPY: %w", err) + } + if s.cfg.RecordHotPages { + s.hotPages.Add(HotPage{Region: uint32(idx), PageOffset: regionOff}) + } + return nil + } + return fmt.Errorf("uffd: fault addr 0x%x outside any registered region", addr) +} + +// prefetchInto walks list and issues a UFFDIO_COPY for each entry +// against the supplied fork's userfaultfd. It tolerates EEXIST/EAGAIN +// the same way the fault handler does so a racing first-touch fault +// from a vCPU does not abort the whole prefetch. +func (s *Server) prefetchInto(fd int, regions []MemoryRegion, list *HotPageList) error { + if list == nil { + return nil + } + pages := list.Snapshot() + if len(pages) == 0 { + return nil + } + page := make([]byte, s.pageSize) + pageSize := uint64(s.pageSize) + for _, hp := range pages { + if int(hp.Region) >= len(regions) { + return fmt.Errorf("uffd: prefetch entry refers to region %d (only %d registered)", hp.Region, len(regions)) + } + r := regions[hp.Region] + if hp.PageOffset+pageSize > r.Size { + return fmt.Errorf("uffd: prefetch offset %d outside region %d size %d", hp.PageOffset, hp.Region, r.Size) + } + dst := uint64(r.BaseHostAddr) + hp.PageOffset + src := int64(r.MemFileOffset + hp.PageOffset) + if _, err := s.memFile.ReadAt(page, src); err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("uffd: prefetch read template at %d: %w", src, err) + } + copyArg := uffdioCopyArg{ + Dst: dst, + Src: uint64(uintptr(unsafe.Pointer(&page[0]))), + Len: pageSize, + } + if err := ioctl(fd, uffdioCopyIoctl, unsafe.Pointer(©Arg)); err != nil { + if errors.Is(err, syscall.EEXIST) || errors.Is(err, syscall.EAGAIN) { + continue + } + return fmt.Errorf("uffd: prefetch UFFDIO_COPY: %w", err) + } + } + return nil +} + +func ioctl(fd int, req uintptr, arg unsafe.Pointer) error { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), req, uintptr(arg)) + if errno != 0 { + return errno + } + return nil +} diff --git a/lib/uffd/server_other.go b/lib/uffd/server_other.go new file mode 100644 index 00000000..7ce7b287 --- /dev/null +++ b/lib/uffd/server_other.go @@ -0,0 +1,15 @@ +//go:build !linux + +package uffd + +import "context" + +// startListener returns ErrUnsupported on non-Linux platforms. +// userfaultfd is a Linux-only kernel feature; callers should fall back +// to letting firecracker mmap the mem-file privately. +func (s *Server) startListener(ctx context.Context, forkID string, socketPath string) (func() error, error) { + _ = ctx + _ = forkID + _ = socketPath + return nil, ErrUnsupported +} diff --git a/lib/uffd/uffd.go b/lib/uffd/uffd.go new file mode 100644 index 00000000..4a4cabd4 --- /dev/null +++ b/lib/uffd/uffd.go @@ -0,0 +1,311 @@ +// Package uffd implements a userfaultfd page server for firecracker +// snapshot fan-out. The server backs many concurrent forks against a +// single read-only template mem-file: instead of letting firecracker +// mmap the mem-file privately per fork (which forces every page to be +// copied on first touch), firecracker is configured to use a +// userfaultfd memory backend, and this server populates pages on +// demand from the template file. +// +// One Server instance handles one template mem-file and any number of +// fork connections. Each fork's firecracker process connects to a +// per-fork UDS and hands the server its userfaultfd via SCM_RIGHTS +// alongside a JSON payload describing the guest memory mappings; the +// server then handles UFFDIO_COPY for every faulted page. +// +// The protocol (firecracker_uffd_protocol below) is the contract +// firecracker speaks; we keep it isolated here so PR 8 can ride on +// top to prefetch hot pages without touching firecracker glue code. +// +// PR 5 ships the server skeleton, the protocol parser, and a unit +// test surface that doesn't require KVM. The hot-path syscalls live +// in server_linux.go behind a build tag because userfaultfd is a +// Linux-only kernel feature. +package uffd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" +) + +// ErrUnsupported is returned on platforms where userfaultfd is not +// available. Callers should treat this as "fall back to mmap MAP_PRIVATE." +var ErrUnsupported = errors.New("userfaultfd unsupported on this platform") + +// MemoryRegion describes a contiguous region of guest physical memory +// that maps into a [BaseHostAddr, BaseHostAddr+Size) virtual range in +// the firecracker process. The server services UFFDIO_COPY into that +// range using bytes from MemFileOffset. +type MemoryRegion struct { + BaseHostAddr uintptr `json:"base_host_virt_addr"` + Size uint64 `json:"size"` + MemFileOffset uint64 `json:"offset"` +} + +// firecrackerHandshake is the JSON payload firecracker sends on its +// UDS connection right before it passes the userfaultfd via SCM_RIGHTS. +// We only use the fields we care about for serving page faults; the +// rest of firecracker's payload is ignored. +type firecrackerHandshake struct { + Mappings []MemoryRegion `json:"mappings"` +} + +// Config configures a Server. +type Config struct { + // MemFilePath is the path to the template mem-file. The server + // opens it read-only and serves pages from it. + MemFilePath string + + // SocketDir is where per-fork UDS files live. The directory must + // exist and be writable by the server. One UDS is created per + // RegisterFork call. + SocketDir string + + // PageSize is the target page size for UFFDIO_COPY. Must be a + // multiple of os.Getpagesize. Zero means use the host page size. + PageSize int + + // RecordHotPages turns on per-fault recording. Every successfully + // served page is appended to the server's hot-page list. Callers + // typically enable this during a template's first warmup fork, + // then HotPages().Save() the result before promoting the template. + RecordHotPages bool +} + +// Server owns the template mem-file and dispatches userfaultfd events +// for every connected fork. It is safe for concurrent use; methods may +// be called from any goroutine. +type Server struct { + cfg Config + memFile *os.File + memSize int64 + + mu sync.Mutex + listens map[string]*forkListen // forkID -> per-fork bookkeeping + closed bool + pageSize int + + hotPages HotPageList // recorded faults; only used when cfg.RecordHotPages +} + +type forkListen struct { + socketPath string + closer func() error + + // prefetch is set by the platform-specific listener once the uffd + // fd has been received and registered. Calling it issues UFFDIO_COPY + // for every entry in the supplied list against the fork's uffd. + // Nil means the fork hasn't connected yet. + prefetch func(*HotPageList) error +} + +// NewServer opens the template mem-file and prepares the server. It +// does not start any goroutines yet; callers register forks one by one. +// When the server is closed, the mem-file fd is released; in-flight +// fork handlers are signaled to exit and joined. +func NewServer(cfg Config) (*Server, error) { + if cfg.MemFilePath == "" { + return nil, errors.New("uffd: MemFilePath is required") + } + if cfg.SocketDir == "" { + return nil, errors.New("uffd: SocketDir is required") + } + if err := os.MkdirAll(cfg.SocketDir, 0o755); err != nil { + return nil, fmt.Errorf("uffd: ensure socket dir: %w", err) + } + f, err := os.Open(cfg.MemFilePath) + if err != nil { + return nil, fmt.Errorf("uffd: open mem-file: %w", err) + } + st, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("uffd: stat mem-file: %w", err) + } + pageSize := cfg.PageSize + if pageSize == 0 { + pageSize = os.Getpagesize() + } + return &Server{ + cfg: cfg, + memFile: f, + memSize: st.Size(), + listens: map[string]*forkListen{}, + pageSize: pageSize, + }, nil +} + +// SocketPath returns the UDS path that should be passed to firecracker +// for a fork. RegisterFork must be called first. +func (s *Server) SocketPath(forkID string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return "", errors.New("uffd: server closed") + } + listen, ok := s.listens[forkID] + if !ok { + return "", fmt.Errorf("uffd: fork %q is not registered", forkID) + } + return listen.socketPath, nil +} + +// MemSize returns the size of the template mem-file in bytes. Useful +// for sizing prefetch buffers and validating handshake mappings. +func (s *Server) MemSize() int64 { return s.memSize } + +// PageSize returns the configured page size in bytes. +func (s *Server) PageSize() int { return s.pageSize } + +// HotPages returns the server's hot-page recorder. The returned value +// is the live list — Add/Snapshot/Save are all valid. Recording only +// happens when Config.RecordHotPages is set; callers may still inspect +// the (empty) list otherwise. +func (s *Server) HotPages() *HotPageList { return &s.hotPages } + +// Prefetch issues UFFDIO_COPY for every entry in list against the fork +// identified by forkID. Used to warm up known-hot pages before the +// guest unpauses, eliminating fault round-trips for anything we've +// pre-recorded. Returns an error if the fork is unknown or hasn't +// connected yet, or if the underlying ioctl fails (other than the +// benign EEXIST/EAGAIN race noted in copyPageForFault). +func (s *Server) Prefetch(forkID string, list *HotPageList) error { + if list == nil || list.Len() == 0 { + return nil + } + s.mu.Lock() + listen, ok := s.listens[forkID] + prefetch := func(*HotPageList) error { return nil } + if ok && listen.prefetch != nil { + prefetch = listen.prefetch + } + s.mu.Unlock() + if !ok { + return fmt.Errorf("uffd: fork %q is not registered", forkID) + } + if listen.prefetch == nil { + return fmt.Errorf("uffd: fork %q has not yet connected", forkID) + } + return prefetch(list) +} + +// installPrefetcher is called by the platform-specific listener once +// the uffd is ready. It is a no-op if the fork has been unregistered. +func (s *Server) installPrefetcher(forkID string, fn func(*HotPageList) error) { + s.mu.Lock() + defer s.mu.Unlock() + listen, ok := s.listens[forkID] + if !ok { + return + } + listen.prefetch = fn +} + +// Close stops the server, closes all per-fork listeners, and releases +// the template mem-file fd. After Close returns, the server cannot be +// reused. +func (s *Server) Close() error { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return nil + } + s.closed = true + listens := s.listens + s.listens = nil + s.mu.Unlock() + + var firstErr error + for _, l := range listens { + if l.closer != nil { + if err := l.closer(); err != nil && firstErr == nil { + firstErr = err + } + } + } + if err := s.memFile.Close(); err != nil && firstErr == nil { + firstErr = err + } + return firstErr +} + +// parseHandshake decodes firecracker's JSON handshake payload. Exposed +// so tests can validate the parser without spinning up a real socket. +func parseHandshake(data []byte) (firecrackerHandshake, error) { + var h firecrackerHandshake + if err := json.Unmarshal(data, &h); err != nil { + return firecrackerHandshake{}, fmt.Errorf("uffd: parse handshake: %w", err) + } + if len(h.Mappings) == 0 { + return firecrackerHandshake{}, errors.New("uffd: handshake has no mappings") + } + return h, nil +} + +// resolveSocketPath returns the per-fork socket path. The server uses +// short names because Unix domain sockets have a tight sun_path limit; +// callers should keep SocketDir short. +func (s *Server) resolveSocketPath(forkID string) string { + return filepath.Join(s.cfg.SocketDir, forkID+".uffd") +} + +// RegisterFork allocates a per-fork listener and waits asynchronously +// for firecracker to connect. The returned context cancels when the +// server closes or the fork unregisters. +// +// On Linux the heavy lifting (accept, recvmsg, ioctl loop) lives in +// server_linux.go; on other platforms RegisterFork returns ErrUnsupported. +func (s *Server) RegisterFork(ctx context.Context, forkID string) (string, error) { + if forkID == "" { + return "", errors.New("uffd: fork id is required") + } + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return "", errors.New("uffd: server closed") + } + if _, dup := s.listens[forkID]; dup { + s.mu.Unlock() + return "", fmt.Errorf("uffd: fork %q already registered", forkID) + } + socketPath := s.resolveSocketPath(forkID) + s.mu.Unlock() + + closer, err := s.startListener(ctx, forkID, socketPath) + if err != nil { + return "", err + } + + s.mu.Lock() + if s.closed { + s.mu.Unlock() + _ = closer() + return "", errors.New("uffd: server closed during register") + } + s.listens[forkID] = &forkListen{socketPath: socketPath, closer: closer} + s.mu.Unlock() + + return socketPath, nil +} + +// UnregisterFork closes the listener for forkID. Called when the fork +// is destroyed; the server stops servicing its faults and removes the +// UDS file. +func (s *Server) UnregisterFork(forkID string) error { + s.mu.Lock() + listen, ok := s.listens[forkID] + if !ok { + s.mu.Unlock() + return nil + } + delete(s.listens, forkID) + s.mu.Unlock() + if listen.closer != nil { + return listen.closer() + } + return nil +} diff --git a/lib/uffd/uffd_test.go b/lib/uffd/uffd_test.go new file mode 100644 index 00000000..c6b2fc3d --- /dev/null +++ b/lib/uffd/uffd_test.go @@ -0,0 +1,127 @@ +package uffd + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTempMemFile(t *testing.T, size int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "memory") + f, err := os.Create(path) + require.NoError(t, err) + require.NoError(t, f.Truncate(int64(size))) + require.NoError(t, f.Close()) + return path +} + +func TestNewServer_RequiresMemFile(t *testing.T) { + _, err := NewServer(Config{SocketDir: t.TempDir()}) + assert.Error(t, err) +} + +func TestNewServer_RequiresSocketDir(t *testing.T) { + _, err := NewServer(Config{MemFilePath: writeTempMemFile(t, 4096)}) + assert.Error(t, err) +} + +func TestNewServer_ReportsMemSizeAndPageSize(t *testing.T) { + memPath := writeTempMemFile(t, 16384) + s, err := NewServer(Config{ + MemFilePath: memPath, + SocketDir: t.TempDir(), + PageSize: 4096, + }) + require.NoError(t, err) + defer s.Close() + + assert.Equal(t, int64(16384), s.MemSize()) + assert.Equal(t, 4096, s.PageSize()) +} + +func TestNewServer_DefaultsPageSizeToHost(t *testing.T) { + s, err := NewServer(Config{ + MemFilePath: writeTempMemFile(t, 4096), + SocketDir: t.TempDir(), + }) + require.NoError(t, err) + defer s.Close() + + assert.Equal(t, os.Getpagesize(), s.PageSize()) +} + +func TestSocketPath_UnregisteredFork(t *testing.T) { + s, err := NewServer(Config{ + MemFilePath: writeTempMemFile(t, 4096), + SocketDir: t.TempDir(), + }) + require.NoError(t, err) + defer s.Close() + + _, err = s.SocketPath("missing") + assert.Error(t, err) +} + +func TestUnregisterFork_UnknownIsNoop(t *testing.T) { + s, err := NewServer(Config{ + MemFilePath: writeTempMemFile(t, 4096), + SocketDir: t.TempDir(), + }) + require.NoError(t, err) + defer s.Close() + + assert.NoError(t, s.UnregisterFork("does-not-exist")) +} + +func TestClose_Idempotent(t *testing.T) { + s, err := NewServer(Config{ + MemFilePath: writeTempMemFile(t, 4096), + SocketDir: t.TempDir(), + }) + require.NoError(t, err) + require.NoError(t, s.Close()) + assert.NoError(t, s.Close()) +} + +func TestParseHandshake_GoodPayload(t *testing.T) { + data := []byte(`{"mappings":[{"base_host_virt_addr":4096,"size":8192,"offset":0}]}`) + hs, err := parseHandshake(data) + require.NoError(t, err) + require.Len(t, hs.Mappings, 1) + assert.Equal(t, uintptr(4096), hs.Mappings[0].BaseHostAddr) + assert.Equal(t, uint64(8192), hs.Mappings[0].Size) + assert.Equal(t, uint64(0), hs.Mappings[0].MemFileOffset) +} + +func TestParseHandshake_RejectsEmptyMappings(t *testing.T) { + _, err := parseHandshake([]byte(`{"mappings":[]}`)) + assert.Error(t, err) +} + +func TestParseHandshake_RejectsBadJSON(t *testing.T) { + _, err := parseHandshake([]byte(`{not json`)) + assert.Error(t, err) +} + +func TestResolveSocketPath_PerFork(t *testing.T) { + dir := t.TempDir() + s, err := NewServer(Config{ + MemFilePath: writeTempMemFile(t, 4096), + SocketDir: dir, + }) + require.NoError(t, err) + defer s.Close() + + got := s.resolveSocketPath("fork-1") + assert.Equal(t, filepath.Join(dir, "fork-1.uffd"), got) +} + +func TestErrUnsupportedSentinel(t *testing.T) { + // The sentinel must be a stable error value so callers can switch on it. + assert.True(t, errors.Is(ErrUnsupported, ErrUnsupported)) +} diff --git a/openapi.yaml b/openapi.yaml index e6446965..454ec151 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -478,6 +478,17 @@ components: Optional final state for the forked instance. Default is the source instance state at fork time. For example, forking from Running defaults the fork result to Running. + share_memory: + type: boolean + description: | + Share the source instance's snapshot mem-file with the fork instead of copying it. + Requires the source to be in Standby state. While at least one fork created with + share_memory=true is alive, the source is mem-locked: start/restore/delete on the + source return 409 until all such forks are deleted (see Instance.fork_count and + Instance.mem_locked). Forks created with share_memory=false are unaffected and may + be created against the same locked source. + default: false + example: false ForkTargetState: type: string @@ -878,7 +889,19 @@ components: $ref: "#/components/schemas/SnapshotPolicy" auto_standby: $ref: "#/components/schemas/AutoStandbyPolicy" - + fork_count: + type: integer + description: | + Number of live forks that share this instance's snapshot mem-file via share_memory=true. + Zero for instances that have never been used as a fan-out parent. Read-only. + example: 0 + mem_locked: + type: boolean + description: | + True iff fork_count > 0 — i.e. at least one share_memory fork is alive against this + instance. While true, start/restore/delete on this instance return 409. Read-only. + example: false + PathInfo: type: object required: [exists]