diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index adcaf05f..8ff70223 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -342,7 +342,26 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original create error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -798,7 +817,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original start error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", @@ -1205,6 +1243,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index d23e7c21..4f04692e 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -16,6 +16,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -46,6 +47,73 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +type createErrorInstanceManager struct { + instances.Manager + err error +} + +func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { + return nil, m.err +} + +// A retained-assignment error must win over the mapping of the create error +// it wraps, or the response omits the instance the caller has to delete. +func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it to retry") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) +} + +func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { @@ -779,6 +847,68 @@ func (m *errActionInstanceManager) RestoreSnapshot(context.Context, string, stri return nil, m.err } +// A retained-assignment error must win over the mapping of the start error +// it wraps, or the response omits the pending vGPU cleanup the caller has to +// resolve. +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + + t.Run("retained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), + "the underlying start failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it or retry start") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) + + t.Run("unretained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 04eb38d2..f73a0ae5 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -185,6 +185,63 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { + allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + if err != nil { + return nil, 0, err + } + protected := make(map[string]struct{}) + var retryAfter time.Duration + for _, inst := range allInstances { + if inst.GPUDevicePath == "" { + continue + } + if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) { + protected[inst.GPUDevicePath] = struct{}{} + continue + } + if inst.GPUAssignedAt == nil { + continue + } + remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + if remaining <= 0 { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + if retryAfter == 0 || remaining < retryAfter { + retryAfter = remaining + } + } + return protected, retryAfter, nil +} + +func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { + protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) + if err != nil { + // Operator-actionable: vendor VFIO reconciliation stays disabled + // host-wide (and releases fail closed on the same inventory) until + // the unreadable instance metadata is repaired. + logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + protected = nil + retryAfter = 0 + } + if err := devices.ReconcileVGPUs(ctx, protected); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + reconcileVGPUs(ctx, instanceManager, logger) + } + }() +} + func run() error { startupStarted := time.Now() slog.Info("starting hypeman initialization") @@ -384,12 +441,9 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) - } + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling vGPU devices...") + reconcileVGPUs(ctx, app.InstanceManager, logger) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba42..85c13281 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,15 +2,18 @@ package main import ( "bytes" + "context" "net/http" "net/http/httptest" "net/url" + "os/exec" "testing" "time" "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" @@ -338,3 +341,38 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } + +type vgpuReconcileManagerStub struct { + instances.Manager + list []instances.Instance +} + +func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { + return s.list, nil +} + +func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + recent := time.Now().Add(-time.Minute) + stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) + + manager := vgpuReconcileManagerStub{list: []instances.Instance{ + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, + {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}}, + }} + + protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + require.NoError(t, err) + require.Positive(t, retryAfter) + require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") +} diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35..1f1a8277 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 7be58b0f..b2838388 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } +func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { + return m.ListInstances(ctx, nil) +} + func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72fe3b94..a4ccd2db 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 9e1d7f02..81533d83 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID @@ -262,11 +262,23 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string - - // Setup cleanup stack early so device attachment errors trigger cleanup + var gpuAssignedAt *time.Time + var stored *StoredMetadata + var retainedVGPU *StoredMetadata + + // Setup cleanup stack early so device attachment errors trigger cleanup. + // When rollback cannot release a vGPU assignment, report whether its + // retention record was persisted. The wrapping defer is registered first + // so it runs after cu.Clean has attempted to retain the metadata. + vgpuPersisted := false + defer func() { + if retErr != nil && retainedVGPU != nil { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -283,8 +295,23 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + stub := StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + DataDir: m.paths.InstanceDir(id), + } + if starterErr == nil { + stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) + } + retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -292,6 +319,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -303,8 +332,28 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retainedVGPU = stored + if retainedVGPU == nil { + retainedVGPU = &StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + GPUProfile: gpuDevice.ProfileName, + GPUFramework: gpuDevice.Framework, + GPUDevicePath: gpuDevice.SysfsPath, + GPUMdevUUID: gpuDevice.MdevUUID, + GPUAssignedAt: gpuAssignedAt, + } + } } }) } @@ -340,7 +389,7 @@ func (m *manager) createInstance( } // 11. Create instance metadata - stored := &StoredMetadata{ + stored = &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, @@ -375,6 +424,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, @@ -574,6 +624,57 @@ func (m *manager) createInstance( return &finalInst, nil } +// cleanupFailedCreate reports whether the retention record for a vGPU +// assignment whose release failed during rollback was persisted. +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { + if retainedVGPU == nil { + m.deleteInstanceData(id) + return false + } + + log := logger.FromContext(ctx) + retentionSurvives := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return retentionSurvives() + } + // Retain identity fields so the instance lists as a recognizable, + // deletable record rather than a nameless phantom, but drop resource + // claims (network, volumes, devices) that rollback already released. + retained := StoredMetadata{ + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + GPURetainedForCleanup: true, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return retentionSurvives() + } + return true +} + // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c..05b3e9d8 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index ad4a192d..b5f157d3 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -152,7 +152,7 @@ func (m *manager) deleteInstanceWithOptions( if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue with cleanup. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 1496ddc9..1abead20 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -219,6 +219,12 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin default: return nil, false, fmt.Errorf("%w: cannot fork from state %s (must be Stopped or Standby)", ErrInvalidState, source.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a fork of it could never boot. Delete the stub to + // release its retained vGPU assignment. + return nil, false, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if !supportValidated { if err := m.validateForkSupport(ctx, stored.HypervisorType); err != nil { diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index a2e7c461..95a04216 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -63,6 +63,31 @@ func TestForkInstanceClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestForkInstanceRejectsVGPURetentionRecord(t *testing.T) { + manager, _ := setupTestManager(t) + ctx := context.Background() + hvType := hypervisor.Type("fork-vgpu-retention-test") + hypervisor.RegisterCapabilities(hvType, hypervisor.Capabilities{SupportsConcurrentForkPrepare: true}) + manager.vmStarters[hvType] = concurrentForkPrepareTestStarter{} + + sourceID := "fork-vgpu-retention-source" + createStoppedSnapshotSourceFixture(t, manager, sourceID, sourceID, hvType) + + meta, err := manager.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, manager.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a fork of + // it could never boot; only delete may act on it. + _, err = manager.ForkInstance(ctx, sourceID, ForkInstanceRequest{Name: "fork-vgpu-retention-copy"}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestForkInstance_VZStoppedSourceSupported(t *testing.T) { t.Parallel() manager, _ := setupTestManager(t) diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918df..f5f49598 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -195,6 +196,90 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +// A failed create whose vGPU release also failed retains a minimal +// GPU-fields-only stub, and the API tells the caller to delete it to retry +// the release. Exercise that recovery path against the exact stub shape +// cleanupFailedCreate writes. +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -239,6 +324,26 @@ func TestStartPersistsStaleVGPUReleaseImmediately(t *testing.T) { assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile, "profile is kept for the next start") } +func TestStartRejectsVGPURetentionRecord(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkNone + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StartInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") + + // The retained assignment must survive the rejected start for delete. + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestStopStoppedInstanceReleasesRetainedVGPU(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) meta, err := m.loadMetadata(id) @@ -300,6 +405,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index bd31818e..765fe25c 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -27,6 +28,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -179,6 +181,8 @@ type manager struct { tracer trace.Tracer now func() time.Time writeFile func(string, []byte, os.FileMode) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error deleteSnapshotFn func(context.Context, string) error egressProxy *egressproxy.Service egressProxyServiceOptions egressproxy.ServiceOptions @@ -277,6 +281,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, + createVGPU: devices.CreateVGPU, + destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -705,6 +711,27 @@ func (m *manager) DefaultHypervisor() hypervisor.Type { return m.defaultHypervisor } +// ListInstancesForReconcile returns every instance's stored metadata or an +// invalid metadata error. It does not derive state: reconcile protection only +// needs raw metadata fields, and hydration would query the hypervisor of +// every instance on the host before the API serves. +func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return nil, err + } + result := make([]Instance, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + } + return result, nil +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/process_identity.go b/lib/instances/process_identity.go index 240d8e55..e44895b7 100644 --- a/lib/instances/process_identity.go +++ b/lib/instances/process_identity.go @@ -171,6 +171,16 @@ func classifyResolvedHypervisorOwner(socketPath string, stored, resolved int, er return 0, fmt.Errorf("cannot confirm ownership of socket %s: %w", socketPath, err) } +// HypervisorMayBeAlive reports whether the recorded hypervisor process may +// still be running. It fails open: when ownership cannot be resolved it +// returns true, which is the safe direction for its callers (reconcile +// protection and claim checks, where true means "protect"). Do not use it to +// authorize teardown. +func HypervisorMayBeAlive(id HypervisorProcessIdentity, socketPath string) bool { + pid, err := resolveLiveHypervisorPID(id, socketPath) + return err != nil || pid > 0 +} + // ProcessExists reports whether pid belongs to a live, non-zombie process. func ProcessExists(pid int) bool { if pid <= 0 { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 65c2b2e8..4c8d1180 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -17,7 +17,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -454,6 +456,47 @@ func TestRefreshHypervisorPIDResolvesSocketOwnerWhenStoredPIDIsDead(t *testing.T assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &stalePID}, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestSocketListenerHelper$") diff --git a/lib/instances/query.go b/lib/instances/query.go index 8e3ed61f..0621460d 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -784,7 +784,7 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb46..ab3db29c 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,34 @@ import ( "github.com/stretchr/testify/require" ) +func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.ListInstancesForReconcile(context.Background()) + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + listed, err = m.ListInstancesForReconcile(context.Background()) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "valid", listed[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 708fd900..ac24db65 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -66,6 +66,13 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create has no boot + // configuration, so a snapshot of it could never be restored or + // forked into a bootable instance. Delete the stub to release its + // retained vGPU assignment. + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } if err := validateForkVolumeSafety(stored.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } @@ -311,6 +318,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index 1549133b..4e3ff48d 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -52,6 +52,31 @@ func TestForkSnapshotClearsVGPUAssignment(t *testing.T) { assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", source.GPUDevicePath) } +func TestCreateSnapshotRejectsVGPURetentionRecord(t *testing.T) { + mgr, _ := setupTestManager(t) + ctx := context.Background() + + sourceID := "snapshot-vgpu-retention" + createStoppedSnapshotSourceFixture(t, mgr, sourceID, sourceID, mgr.defaultHypervisor) + + meta, err := mgr.loadMetadata(sourceID) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + meta.GPURetainedForCleanup = true + require.NoError(t, mgr.saveMetadata(meta)) + + // The delete-only retention stub has no boot configuration, so a snapshot + // of it could never be restored or forked into a bootable instance. + _, err = mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "snapshot-vgpu-retention", + }) + require.ErrorIs(t, err, ErrInvalidState) + require.ErrorContains(t, err, "delete it to release the assignment") +} + func TestRestoreSnapshotDoesNotResurrectStaleVGPUAssignment(t *testing.T) { mgr, _ := setupTestManager(t) ctx := context.Background() @@ -113,6 +138,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +153,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c83245..44dc5021 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -48,13 +47,20 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPURetainedForCleanup { + // A delete-only retention stub from a failed create: it carries no + // boot configuration, so starting it would release the retained VF + // and then boot an incomplete record. Delete retries the release. + log.ErrorContext(ctx, "refusing to start vGPU retention record", "instance_id", id) + return nil, fmt.Errorf("%w: instance retains a vGPU assignment from a failed create and has no boot configuration; delete it to release the assignment", ErrInvalidState) + } // Release any assignment retained by an earlier failed release and // persist the cleared fields immediately, so a failure later in start // cannot leave on-disk metadata pointing at a device that is already // gone (matching releaseRetainedVGPULocked). if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -64,6 +70,13 @@ func (m *manager) startInstance( } } + // Do not persist the previous VMM's identity with a new vGPU assignment. + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" + rollbackMeta := *meta + rollbackMeta.Phases = meta.Phases.Clone() + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -111,6 +124,16 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + // Registered before cu.Clean so it runs after cleanup and can report a + // vGPU assignment that rollback failed to destroy, matching create's + // vgpu_cleanup_pending contract. + vgpuRetained := false + vgpuRetentionPersisted := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -162,27 +185,34 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + assignedAt := m.nowUTC() + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} + } + return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} + } + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) - } + vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 9076b715..dca79ec3 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -227,7 +227,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if path := storedVGPUDevicePath(stored); path != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "device_path", path) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "device_path", path, "error", err) } } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index a293fc6e..dd932d41 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -187,8 +187,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files +// listMetadataFiles returns paths to all instance metadata files. func (m *manager) listMetadataFiles() ([]string, error) { + return m.listMetadataFilesWithStatErrors(false) +} + +func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -210,6 +214,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/types.go b/lib/instances/types.go index ff998bc8..8b442c18 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -153,7 +153,12 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection + // GPURetainedForCleanup marks a delete-only retention stub written when a + // failed create could not release its vGPU: the record has no boot + // configuration, so only delete (which retries the release) may act on it. + GPURetainedForCleanup bool // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6ace..d0cd9b22 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,41 +2,220 @@ package instances import ( "context" + "errors" + "fmt" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a +// persisted hypervisor PID is treated as potentially live. +const VGPUAssignmentStartupGracePeriod = 5 * time.Minute + +// VGPUCleanupPendingError reports a failed create whose vGPU release also +// failed during rollback. When Retained is true, deleting the retained instance +// retries the release; otherwise startup reconciliation recovers the assignment. +type VGPUCleanupPendingError struct { + InstanceID string + Retained bool + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +// retainedVGPUFromCreateError fills stub with the pending device's assignment +// fields when err carries a failed device-layer cleanup. The caller provides +// identity fields on stub so the retained record lists as a recognizable, +// deletable instance. +func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return nil + } + stub.GPUProfile = device.ProfileName + stub.GPUFramework = device.Framework + stub.GPUDevicePath = device.SysfsPath + stub.GPUMdevUUID = device.MdevUUID + stub.GPUAssignedAt = &assignedAt + return &stub +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The +// cleanup stack is LIFO, so cleanups registered after this one run before it +// and this restore would clobber anything they persisted; it is safe only +// while no such cleanup writes metadata and the instance lock serializes +// start. Violating that requires switching to targeted field restores. +// +// It reports whether the assignment was retained after a failed destroy and +// whether that retention record was persisted, so start can surface the +// pending cleanup as a typed error like create does. +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) + assignment := devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + } + cleanupMeta := rollbackMeta + releaseErr := m.destroyVGPUAssignment(ctx, assignment) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) + setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + retained = true + } + if err := m.saveMetadata(&cleanupMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if releaseErr != nil { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + if !retained { + return false, false + } + // The mid-start save may already have persisted this assignment, in + // which case the on-disk record still points at the device and + // delete or a retried start can release it (matching create's + // retention-survives check). + if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { + return true, true + } + return true, false + } + return retained, retained +} + +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, + // Vendor VFIO VFs are reused across instances, so stale metadata can + // point at a path claimed by a live instance and the release must fail + // closed on an incomplete inventory. mdev UUIDs are unique and never + // reused, so skip the scan there — it would let one unreadable + // metadata file block every mdev release on the host. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { + return err + } } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - return err + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { + return err + } } } clearStoredVGPUDevice(stored) return nil } +// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's +// stored metadata claims devicePath. It reads raw metadata instead of +// hydrating full instances: the scan runs on every vendor VFIO release, and +// deriving state would query the hypervisor of every instance on the host. +// A confirmed live claimant returns true. Unreadable metadata, a recent +// assignment without a PID, or unverifiable process ownership returns an error +// so the requester retains its assignment for a later retry. +func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return false, fmt.Errorf("list instances for vGPU release check: %w", err) + } + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + if id == excludeID { + continue + } + meta, err := m.loadMetadata(id) + if err != nil { + return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) + } + stored := &meta.StoredMetadata + if storedVGPUDevicePath(stored) != devicePath { + continue + } + if stored.HypervisorPID == nil { + if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { + continue + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + } + pid, err := resolveLiveHypervisorPID(stored.HypervisorProcessIdentity, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + } + if pid > 0 { + return true, nil + } + // A dead PID with a recent assignment gets the same bounded grace as + // startup reconcile protection, so the two guards agree in the + // fail-closed direction while a mid-boot claimant hydrates. + if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + } + } + return false, nil +} + // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped // instance after a failed release during the original stop. It is a no-op // when no assignment is retained, and a failed retry only logs so the @@ -52,7 +231,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { if storedVGPUDevicePath(stored) == "" { return } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) return } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c4681..1d0e4e3b 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,17 +2,642 @@ package instances import ( "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + assignedAt := time.Now().UTC() + stored := &StoredMetadata{ + Id: "failed-create", + Name: "failed-create", + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "mdev-uuid", + GPUAssignedAt: &assignedAt, + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir("failed-create"), + } + + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + + retained, err := m.loadMetadata(stored.Id) + require.NoError(t, err) + assert.Equal(t, stored.Id, retained.Id) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) + // Identity fields survive so the retained record lists as a + // recognizable, deletable instance instead of a nameless phantom. + assert.Equal(t, stored.Name, retained.Name) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.HypervisorType, retained.HypervisorType) + assert.Equal(t, stored.DataDir, retained.DataDir) + // Resource claims released by rollback stay dropped. + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) + // The stub has no boot configuration, so it is marked delete-only. + assert.True(t, retained.GPURetainedForCleanup) +} + +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) + + assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) + + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) +} + +func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { + t.Parallel() + + cause := errors.New("boot failed") + retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} + assert.ErrorIs(t, retained, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) + + unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, unpersisted, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) +} + +func TestVGPUDevicePendingCleanup(t *testing.T) { + t.Parallel() + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("rollback failed") + pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + + wrapped := fmt.Errorf("create failed: %w", pending) + actual, ok := vgpuDevicePendingCleanup(wrapped) + require.True(t, ok) + assert.Equal(t, device, *actual) + + assignedAt := time.Now().UTC() + retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) + require.NotNil(t, retained) + assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") + assert.Equal(t, "img", retained.Image) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + + actual, ok = vgpuDevicePendingCleanup(cause) + assert.False(t, ok) + assert.Nil(t, actual) + assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) +} + +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} + +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + return &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, nil + }, + destroyVGPU: destroy, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: hypervisor.TypeQEMU, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + return m, id +} + +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + +func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + cause := errors.New("create failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, cause + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, cause) +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) +} + +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + // The mid-start save already persisted the assignment. + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + + // The cleanup save fails, but the surviving on-disk record still points + // at the device, so retention must be reported as persisted. + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + return nil + }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + previousProgramStart := previousStart.Add(time.Second) + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + Cmd: []string{"old-command"}, + StartedAt: &previousStart, + ProgramStartedAt: &previousProgramStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + + partial := rollbackMeta + partial.Entrypoint = []string{"new-entrypoint"} + partial.Cmd = []string{"new-command"} + partial.StartedAt = ptr(time.Now().UTC()) + partial.ProgramStartedAt = nil + partial.ExitCode = nil + partial.ExitMessage = "" + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) + assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) + assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) + assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) + assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) + assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) + assert.Empty(t, stored.GPUDevicePath) + assert.Nil(t, stored.GPUAssignedAt) +} + +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("booting-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "booting-claimant", + Name: "booting-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + assert.Contains(t, err.Error(), "booting-claimant") +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("stale-claimant")) + assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "stale-claimant", + Name: "stale-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + claimantID := "claimant-dead-pid" + require.NoError(t, m.ensureDirectories(claimantID)) + deadPID := 1<<22 - 1 + require.False(t, ProcessExists(deadPID)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + // Same bounded grace as startup reconcile: a recent claim whose PID is + // dead fails closed instead of being treated as unclaimed. + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + + // Past the grace period the dead claim no longer blocks the release. + stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) + meta, err := m.loadMetadata(claimantID) + require.NoError(t, err) + meta.GPUAssignedAt = &stale + require.NoError(t, m.saveMetadata(meta)) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("dead-claimant")) + deadPID := 1 << 30 + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &deadPID}, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") +} + +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "legacy-uuid", })) assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ @@ -24,11 +649,12 @@ func TestStoredVGPUDevicePath(t *testing.T) { func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} stored := &StoredMetadata{ GPUFramework: devices.VGPUFramework("future-framework"), GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) + err := m.releaseStoredVGPU(context.Background(), stored) assert.Error(t, err) assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -37,18 +663,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() + assignedAt := time.Now().UTC() stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", - }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, assignedAt) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.Equal(t, assignedAt, *stored.GPUAssignedAt) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) + assert.Nil(t, stored.GPUAssignedAt) } diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index bab6f06d..a4246479 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } +func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { + return nil, nil +} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index dcff0301..15ed6a8b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1345,7 +1345,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -18997,202 +19000,203 @@ var swaggerSpec = []string{ "YLywGivLMDWPmRoL+1oApPOPQn4/vj8ajUb9XqHqHNp/j0LU9FG9g0MHQotJsQCHJxwcLeDsqVyQXMRM", "FfWXGZJDM4bshp7F90Sf9Z93E6/c56VcFT6Y28Bqu6HQvi/0Jwy1RYGF8MlbaK/3WtW1rZy0U1KOl19d", "Oo77anITFz8jkcyT2KpEU3vbocWKxc5Op5lB7ozvck1eYe3D+tRdbK6R5PecqRV5fXZWiwtQbGZ5QLeJ", - "A5do2QeZ3WgbDrYYEbaO5paAsHcBAtuUVCoS4geHfK264HyWMFJoB1dcqUgGMyu4wK2xdLJhTg3zdsyW", - "kzwPKSL2kccGefXq9EmNOCi9v/9w9PDR4OF0//7gKB7tD+j+4f3BwT06mh1GDw73Dw43ZGF0SOW6fXZW", - "UBsNBNEWIdMTH7odiqhtC5xv3PsuGPiai1he166WYHRmtXcX+bmt+/W47s5DCGaDJFQbNA20cIkzuEBZ", - "ZNvGaGiXLuhNkC1WtvsvR/tbrGxb+QUMroX/vlS5QJ8eJrgX9uu0MuDqZtXHeTt2CgPyWRfbVqvaefdF", - "Gx3fe3R8730XzWcObBtjk5zucHPbwpE8AHAjNcGnx1VsNB4ws+fkDTQpu0yGXr9XJFvA33DRNgJ5i8ed", - "MojaDmw/zEY28e+WzN3Tmi4AYRIIABcfWynAy/oAs1/kh1vx4iSReUwqdi7EwwIn0GlFL7DNEKxqD+Yv", - "xLfETACrPwCgMsD5c2EZMTi/bCMuzfeYPIN34RFNUWVyg8CiElW/D41XGKxhz5fvGhWYzUO+cLoLfGMV", - "GWL/BdO2y+DMoZubQMnnmPwi4ZtCkxKyaVfF10GFWX+9aYPdcVjRHrYBOnNi3DH5oRDdCuHPCXs7mrk/", - "J45hlWgpu7WcdbfjPUst5c5V8q/7PVzRXr/nFwrytNcztl+VVL92/qqkGIpiYjSBs1wmqOaG+0LzMBOu", - "DY+0y1ywm9smX7h6NyyeoGLSFhCJWY9OeSk+8uLL6zOyAzCAfyHOaGz/tVsET9buuoNHR4/uPzh4dL8T", - "2E85wO1i5wnk5K4PbqsMGmX5xNkd2qZ+cv4K7QoRauxF4MXrsyq2QqakZT125r7BauePho+qGEexzLFG", - "vRuSA0RDSFXYsCCMV8GLWoLwfufJks9m4ve30dXB3xVP99/c1wfT/RbsVuwobNI6rXq21+y/bDrA2jVh", - "GBogKKVbkZpeMA0zIBfMEKCfAaERqA5FKq0jOY/n5FY8SFhHh4eHDx/cO+hEV250lYMzAQNX4FJ2I6gc", - "MXiT7Ly4uCB7FYLDNj2gAcBrC6dWhs8ZcfVbR3WBdLg/OgxRScvFXVKNa3uZti75a6eauUm5RYeM4EJt", - "WzvlwdU+PBw9OLr38F63Y+xMrxP1ZjOH8fkyuDwOBb668zsgTb58fE4gG3VGo7rdxIcn3WhU5kajggoG", - "iDx+g4E9fHD/3tHhwX43yLFQxIMD06sd2DrvChy6AFEEdiOwFOust992W4TEKSSwFyxKKE8fRz6+v3H7", - "IML4ROFr5SZ0uRictX/t4urwbSfDUWEOwuwQFA2kIrko6loMt7s7P4jXsp1r4/WwnauHckKEXT2HjYN1", - "rW6xlJliSy5z/QEakgYTNmeJlOpG37YpLC+YzhODrkWuyeuzb4CnWFoj2rCsrkM5atyAIHTLyd3oPNdI", - "JEzkbYvVaTe6bP2mCfdbTm1/E5pDjRu04nbFlnPlYnvk4QlNohwqudBiP+2sAIAK8tGzLFlhYHmSSClI", - "tKACHBCqArdDFjKJh8EwTPtkMgtGLMhrkkhEHL5iLHNFTnAQ9jMrwvAlIzuVfHiCpNQoRnkvRSbjyljU", - "qfFeGq6qR3UoU6rIw7brSY2sgOHiJzWTYyLnGpRCAyHzwyYGe0YVRsJTgUV7linqkoFo38AQG8w8dKPi", - "TSpnTsF1IgdkOeNK0khJrQlL+BwKxLw+ayTPbki4KlJot0cT1gfbgXTRWRi4yhCKqXNtr9D9GEgmeZ8b", - "EmgYEtY2xOl542RKRQ5lTyqEzN5kXCF5dIvFW0htJgUo0g0Hq80EahnkipVQbUXKd2EP8u8E70XP2m6z", - "XC7o9VZfr1FVuKm2Abbz1OCKhlerX9BgiIzXYaE2IlGV0FZNHKObIKWV4PdcQ6u8gplFdiDhosKWKvhn", - "u10CUMIqq+1nTVt1JRt/PhpddMUU2wwhdk7N4lTMZAB44gZeSW+JdqGSGVMph2oeJGaCs9jrkoV70pm6", - "IFs50YzEOXMrh/Kpom7BKR5vAI8Q3kbGxbzB65sddjEP4xg2lzqAft2LXUKJdDib86XKYa0wFlATWuZ1", - "dgqw5HoSdmetN6zYPE+oIg4NsMuQ9SpNuLjq0rpepVOZ8IjYD5o+55lMEnk9sY/0dzCX3U6zsx9M2uri", - "XODgXFIabkij33IK39lZ7jZSYsESs4ff7wFcSZfIrGCY8g88YQ5a7pXgbyqEXscCPzoYtaVqtzRaS9Je", - "hyW8Ked2JBs88bkOJLZtlHJ8RR8WO4B2FHuyXGNxkQ63kkcA9S7A23l06lkL74eLcYL8uoGKQaYMkk78", - "1Na5Rge22GUqwboGuV6Qv8tp3SDaNaQ2UC1rR5T4DIrNgsHlsKMbDdL4xtqaVHb3JgAIwFbtROGjG+IK", - "bKsrVsYutfGTF2slthbMLRn3c8RyWx3KR/jYiiJ33vXaPYm+WQwtEAsMECnarKCsKNRyWVUq/GkylUoB", - "/LGVcKTwswHMDyvz2LX2oEvk5YKtiGIp5WIsuCiMpIDkxYhgS6YqKZpSWSVrzuIh+VtFxQPA6DQzK4dE", - "DsbzbzSR16IY41hUB2kbz7Vt57FAy6LKoYJ7+ZJtFrQ+SyiQMgtOMKOgeB83CzJTTC+qcw8VbLQy3rVU", - "cWslnBXxr0CBFfCxEiOvmKiysqKZoGqIDU3wq/UIOay2Ck+d/klqBVBJs8Dp5v5yzVRYSCymVLzSKXSl", - "clQqygkikAAcBxS2c38hiy8gODoAbpTN/9U3Wf50XjRe/63xWgVUw2PcPkazbdAEG2GKTCPYp+5J2xqq", - "AjlYm6BU1n0JZMeHJ/vqIHVJoFKlo9M92S0NrBmI70ezp1lU7/3o4b0H9zuWSXkvZx1CR31o19wy3eCS", - "a9mpsy5+n4f3Hj56dHh079HBjTwsPmejZX/a8jaq+0N22BtjD2vyr3/88/VZw+tzD+KbRzcaFGZthIfU", - "krlRH9Drs3/9459+VLceUIjRrMNTt/jtW6N0kupO+kCBuguvm5Nsg37/uGYkoAWbITtsNmNgBp3gug3K", - "wTQwKLpJwTSjETerACOn1xhJXrzSgFnu4g6qDzYk8mLbDpLTci6dT8uMxx3fOflP9A03aOFh52pLOp+2", - "+aGfN3tFL3TptajGOHQIMdBFoe91A3cxn2uqa8HS9u8Ichp89tZ6Jgu+sRnytZlmAFEsrqhYJRQwBBXe", - "kCfdR9Xtb2xnxW9ZM+s0V/y3Deew/QjeyOobuJEDRt9oe9pqgz+4C/B2X02m1TpoGwvN1Yqmlbfuzfvt", - "kJm7DtJf3GA376+SjHiTD5uAtECPbgxuycu2+zWSaKGmSp5JwIAmEzYoAvVcEgrROXoE7Zl3GOeB7Mjo", - "Ss5mdaDVe+3A3IA5A4lUvhdqjNVM+oS98TaLJqozAsyMe/f0uGdVgHFvPx33Gm6rYGpiSt9MXAd1YJHR", - "JqTsogJoc5Daz2CayOgKS3xB5eghGZGUUaFJLuDwN7xq+6PN3qF+L6vsTYFLzTDEaY1twZimbEGXHMoh", - "OJ/KvBaIyd5woyFgFNo5JrFEqKFafVM3Q/saJg4el5OGS4eKlWvYNmjfk8JHtJbvgoFvBlVVxVumZN9l", - "yluO/fz5WR8DGCD0EAdWi2/0E8URWAZZdNHA9i9/D8cPTxM2gXE3seLT9XWs5nuDZ1UxzYx24NElOTSI", - "gEQyF6YJIp92U+TqKVvrV1IuINjPhX8AKJjr3RXHj1kEJ1Kvn8U6od+CuBt5A26lQ4kDhyEShkMBvqSw", - "r/iFcwg3B4DGhkppYmynGteNXsKJNtLVsipO9YS9iRiLm2iT4Ve6xsq7L4Ox8j9TB1BTVA12b0O88/rs", - "hh8veQrG2rba1Zh+IcUAoDH8ljoYC8Slc0ApdUKr4V5X4CImIWzP0AtdspnZm81r/Qt7YwCcO84TRFwL", - "k65jVe4y2rbit84abDvQUm0vsv8RaqZhvPmtqqa5UPVPUTjNvfVRiqWt7c4FM/7dC0dG7SXza1VGai4t", - "H/DvX6nH2CAp9Ym74Ml+utsgwaNF2CriEGE75j8KmrJJptiMv9lAPPgCKsZ1yJDyIBUZDAhuuZPSN+To", - "AYkWVOnG2AWfL0yyqgfgHAWAfN6roqBihglvKOyy8+Vu+g/Xo93cdlZbDwnHFxXYnbV6Gk4knWwCbT4p", - "vW3OOp/RFVhxWp2EDw6PRqPDg9GtUJv9sG6wXCflJ67+Xr2dtpS6ynfO0V+LUq22UCQwrxd1vVYc8qCL", - "ZdJGMZoeQ+JNRiNGEjYDhLaimPZ2z2Kz682DdwKVA3sp6N9vlNs374Ov12spunKA134aPe9crOP7VJ9v", - "cYi2sJloDc8tkHN3OBjdf7l/eHzv/vH+/sdAWi4WqS3b48Hb/esHyQGdHSUPVw9+3188mB+kh0E97Ipj", - "WZoutPqTfbc1yqa8JOs4QTWWRnbcHDKmmtV6m1WuNUu4YANdZEhtT1PcwAvQ/771/N/Mzo8z2Cg7XNQn", - "WRUhqCkXp0ZZnwbbyk1mo++iOZvTJ5tncasMpOZAwvTWHAqQV7fBQHmE/d57oh7kouM19KryYueLaGNW", - "3LarKORhh5Me3OWWFQ+Rd41P1g7cpgt8/ZIL2E7nUnGzSDffFsVrBYY1xE2/1SauYykNyelcQKnu6s9F", - "mFxVibIf9/q95O1R/cy437ujajn424IA3VZXpYIOYWRQCX7zKsArpeKhMJLd6up2zN/tD/YfQRxC8vbo", - "u9HgUT3ioI+rVV2+ff927ddRlzWs1p/zdYv2H90o4tqv5yYK+omHqqeV97IDxnU0XhZG9leHT7itbXD5", - "eG2PGyg5rQLo+0p67nKbVIWmmCV0FQJGrxhqdUN7rBIZmbI5F7qL3fZwVBhu76Xj3pA8dujUoMuWZfBr", - "zUMB9Aqd8DRlMbcyJqr+7RkMBx1tcU1d4maFMfxXAWltGBbXHm2HSNiWcLXtmhy+Rz7ue2m/3TTeTegd", - "YFfzKirgc8GLfcJnhIpGdUwuljThsUukh8RIiFc79iBoJck6HqBLOdDbSfpkLg0pU+g72tty0W4XLMbP", - "3oC9dQNmBhLEwQcBRCnAufgm9nX6hGRKxnlU5o8mMOgS8UPlDfizDUL+9pDcj2nfgMTsmVRku32jzaDR", - "zT7Ztt8N26Ql2Pat3h9t3+qPYhTp9/Is3s7D8KVuHOxGsOFbUhADJpr6sjckwcpkfuvA0V9UV3Bd50Vb", - "cmRFojzzDhZLU+uUFHC3gIshFNf7hCXMXlPrjRCZxGWWBNclF93OUvfvP1y0uTjBI7U+kJ8Yy6yuAvhH", - "0F9KxSo4MF/zsrhLdka+nq1Gh9cAa+W41aoP7sFWSax1q6om3Db8fuTyDZs34qVcVczfBX52VTRbR0Dx", - "DL8mpL1ox593X/qwt1b78ccwy31KIe25cz00IFE9YG+BPO77L2OBrVhXJ96jkHs+RBYvnWbcBvPazAKt", - "Wp0fD/4HrcxkMjze++4v//fgt/8MWpsberNmahCzGQQaXbHVACvfWB19WAc5Bdh9K0zPHakwmoINCQDE", - "3WGsjvfeqGAaq19oujYFiNCqlK3Z3zqhv/xHe3xTZRlfAZ/cSrLvXZXiY1TvNNJfRzspU3MfS+4TyXaH", - "YwG5aVdspUmlGJYTaTyhfqOLTyoR6OQSxcAhE8tLMuVQXVCPhdVqaRSxzGoTrr4KxxLZEriPYjSptuOK", - "cvnEb+eQxHgCRl6frSHkPn/18vvnr355Mnl+/vSXx6eTn57+N4R4XA+wh3hgae/o3n1XGLu6kvuBLX4P", - "DP/3AqkNkRtCXQboC5Isodp4QGDmGiASfLhB5WWyA8kKvtKmz9XcvRn05uOiwWCw2wcuqDJ69CGKHr7a", - "WOVwKZOBlahbgO+DBkxci2CwNjSFQfC9Njv3fBoQG501cc7nNGDaDpmwP0hxQj+grUk6a/vfWqUpHDr/", - "pFkVALkBLlUDxb6hl2ozaI+sT60gNSkrsNfjNXLh0iV5JZyrnhuZCrPnioiGIBpiCdjUmxJky1PmAfkG", - "8NH2vM+NonxlZpWRtO/NmZdYGzL1hgU6t0tzvWCKVTYCPijR1m+4ZC4VpAPwBxZhy5gqwyR9HokVhMDD", - "qclOYWzwS1AkuK5bYDej6Z/RN0UPYL2nes3lBfMoy93sP/sekM9f+BJ/fOabgGE09Ikwznedijatiaeq", - "9c2oUtX6vPH94MFzvGoD92s7Ww3iLPuokWaIHv9GuflBKtBA2mE2PjpcOGg3MVMAO9YEA++EpM1TFk9k", - "bjaff1dB3GFsFGUgyzKiXtuiQMRRLYO0jRd4IIhyDOsrbZeDRbniZmXVcycMTyHzztfuhIWEjuDnsmOo", - "l/juHdgpZ4EchWdMMMUjqEZpz2NKBQjp5PVZpSgZ1qdbgwgF8fL5yanTcD3KLGgs3ADp+VC/x+envX5v", - "yRRqeb3R8HA4gsOcMUEz3jvuHQ73h6MeCPILmOIeVBB3KbsuxbXQlU5jJwl971+yXyqaMgNf/BpIPodQ", - "N/c6SL10XtFbMsqVU1yyBJLikWC4/RrQ4/2Feoy3ch+XvbOZDjJbIeGCZc/d5v4GQiWcHZjmwWjksLSN", - "u34hXQRj1Pf+7gIWy347SXVuiQJg6muahZcti6V/1+8djfZvNKZNQ4GzG+r4laAub5SBQnjvhgtxq05P", - "BWaCubxeF4FTPXFASNWz9utvds90nqZUrfyCVVcrk7pNMGaaUP8uQvwaTSLLKqDky5A8F8yVzaeGUAyW", - "VbmAUrL+Q0uh9VOAbftNLnBxvpfx6oMtYa0Prxa/q7Mze1zerdHzh6OdgozXN9I98qDOSLV3QEDf06IO", - "8ic7KUejRx+/0xMpZgmPDBkUBOxCYLmGKJMEIKo93I1U5PdcGkqKCPIv6Eg7mXVakFu/vIr2/uDxOzze", - "CQtZXs+ZSqnAeHx8Z8uhXzvOaAUvj/PGW80TPlSTgJvK477gRQWCXP2IVq+tpjC4fh0dBZL+XZ84vfgT", - "Ev7RHZxwN9miFOanPHJQX5Hkmn1Jx8l5daalEBKU5Z4x87nQ/OguryyHW/8nPEVfCgE/Y4WEV+7W2qWw", - "l6lcoAIclABflDly7rtv6sLfy/JJJTADTOm2aaigYNCVR+PVkPg1RaXfrADVRzGYZ7x+rZzb4X0uJ+zg", - "Lk4YzLhwTny9pr5eU5tOOVKLnwIczMop72CDuJEF4s9nf7ix9eGr7aG77aGT5UGwa2dd+LucDokLgoQ6", - "9Hoh8yQmU0YQYseHOxiqhvO3hKpowZcMcNSgLlieGJ5RBcEMKYmpoei2bTVMbDRLFM3t2eYGPvStXOAm", - "dIJmE4B+m7RBHpZBb1wIFhP7iUOLKxHs1qpD49kPGtiLBsurkVwvpGYFhJwwldscMmo1asfQ7HAsXjps", - "UbuAEL/reY1mCSCkbrD/SEHoWLgPvvUsxMceaZqWnIsqgKnjCIaI27KeTWVHOtGRDMG7vGSCCjPQGYv4", - "jEduWlds5UIIgw12KvVjB+zH+fqsyBEgB7thiDBABAzjwT4pnhFHSXX/jYC42yjJ49LJ5VFrqJrSJAnW", - "gpgnckqTCa7PFQv4BJ/BG25RqlXkvTdJyJhhRfBsZRZS4N/5NBcmx7+nSl5rpsa93eFYQOy/W2sW90sB", - "kVxD7bA0k/acKZlin3s4xL0/rtjq3XAsHscpF54i4BOaaEnYG/gOSioBTANyrxZ6wNMU9oOf5NrItAq2", - "6ekOhylzk+XGJTFoZvohoMmxMJL84eEE3+39Ufb4DpzFjMaWTiqv4JRAtm4btZ5QO/sJvBpwtzNYgHHP", - "XqTjnv17rqgwiBRZ4CGSeXVLdwpAfntId5srHFFBMplhMQMgqgW1JFdrA+ABaJIQA0fJf2sFd9jJlvk4", - "tLd02gr1hthcjWPEBTn7vnKYRkcPw+dJs0ixUETJf108/4XArWz3AF8rI4Qwi0BYgYHEObhOPU97SqMF", - "QUcV1K8b93g87hXu3HgXxpprlzA/GIBP8Ts7tO+wmz6PvxsObVPorjwmv/6BrRzbs5SlCD057r3rk8qD", - "OTeLfFo8+y28oG2IWRc1RkB28JrbBU5COYCbVG58vCKpiIl0t0CyIpSUHKgauDLlgqrVpty1wNK7FZQz", - "DJ6rLMYfYwiWG/eOxz5cbtzrj3tMLOE3F1M37r0Lr4DzWrYXS4P7rHBuFkR0fzTa3Q6+7NY34LPs4Bj4", - "wDpgq1ZUVHq0O+iQP/9c/oF/a/2zcP1QYTsv0XBQ8ffG9y/QAVGR2KuaaMAF0RC7qYhY4sXu7Yaeu3ce", - "2M2KWJLcNYF+KvIs3GMFOPwXRY6wWeUx2mi+/8QUN7qrS6Vmtv809PvF2c8D1nNnO2dLH+ocLo0BsCdO", - "lSb4MqGaXMCYBhdW+X4Kvw7df73uBzB+l4mcXx6j6k4SOScJFy4EvRKobMUDt5bwESKfFN85IBRfl2wH", - "JYl//eOfMCgu5v/6xz8dnPi//vFPOO57iOgFZY0vF4wqM2XUXB6TnxjLBjThS+YnA4VH2ZKpFTkcOZs/", - "PCKV6upOStNjMRYvmMmVqITqY4kw7Rp0rgI7Hy5yph1yjH2Rz1z9EoxtDNht/FnGpbzTE90PIPDBDCoT", - "sLeipwGAL+NY29lpor2wyRTnXDOaNsM014L1tvMXw94YpN4BDvCGDAaWOHTu4IGbNNm5uHi6OySgbSFV", - "QI0a0B3KZpwaMfzKk7bzJOQodYYCq4y8KaIZnfKEe5NjS4ENPIIpjRZcsDK+uIC39k0c+5FaHvP4/JS4", - "QMg+vDoWzy/2wMRqWGRyxfqOEygHallW4JIu/RF6AP7FDUSHDdy7YzFjFLAWTp8gE6jgPhcpaEXDArAj", - "IMYVC9YXXfXHAsFLHViuPXipjFkCH0H/c2rYNV31SVFe1RfkSKixCrHu25fHAuFF3RoMAB2DVIY5BH6G", - "Qxr4SF6XJqTYLLGqMUTgY6Vp6HtnJhVxEc6VwvK+O8zrw2HZRUtp9PzCzm8OmqBEeyC09PzC78Zun2hJ", - "ooQDNURUjMUcAoE8XqwUtV0tcpgWVMWDSNpLoIofdCXkdcLieRuPPakS2UeUZGr9BI7Tj01y/dKEi8X6", - "BOwhRky0zZ67J+6dbq471+KfyXfnag/ewHmHFlyG/AZX96sjr4MjL7xu3qkX8qw98aB/Hy/iF7v4RAG/", - "nvbW1xyfVJbsU1j0yI5HUwGviFTk/OSU0DhWTOvdf297n50pUmkp/9n70bLiTxF64sYilcOZc/aWOoF8", - "KezghRs1oX5ezZKu1fttr1bvpfWmK0q/lFfex789Gp3e5Bophd6S1r7eJFuDbbmOJFS2K6llAKJRwgrx", - "pTinVSraZlXGMN7iytkoLjn2fPrEH8i7sy+7rnPRvBvugCk+aTDET8gI66nW1ULNXxI1vyp20QMcbzA/", - "f16kObo7KeiuTdEhMv+S1MW4sWyWCyIoSOsF+owZhAL5mHq66yEw8Qum/Kn2dfVh1sW08FOCmCYwIbDE", - "bNZ9T/GVbqovtvdn0nxheW4isbgl/yqidFB2y7XapOCeuqrDH0+/hR5upN5+uLAVR2CBRQYr6tS7ncCy", - "ukP1SkS7XyNXPjhFY1xjqcQqP28WF5ZsRFMq9Ky7kusei2qJayvTOb2WCzJL+HzhnAAxn0GsnqmWjIZR", - "HtzBKIvSzIoa5kIUv8S833O7yM4LvGTKkOcnp7j+1St17w8IWt2uKnnmtfF2ffXi5wETkYwL50m7TOqe", - "fGCFCem/lst796fuC8xn5V48aBMY32P/MZicYPz7kMv/dfBDwqeKqtX/OviBJhkX7H8dPk6oYdrsfjRi", - "Gd3VTXfXCswXTHxWf+H1RQPWJOaAUrpF4C/e6ijz+/f/VGI/TvpGgn+xrl9l/y6yf3W5Nor/bis+qgKA", - "fXwiD1dBbKHVhkdfIW3uwGjqKLICaVPzIpWgNgupDTz68vKbXVA5Lyiuem10tP6XB3Lj9eFJ9/RJHxYS", - "ihhDEQWXPnhHvgA/jjsXbl2/d+8IeJxO+TyXua5mJqbURAumXdZuwuoM+EsTu8vruVXw/oypdHSXV8ed", - "y9Vf6f4jSfzNDUXmjQ69bTK/f6urzO/etzI/Ipq6zGZX6aHvqwDttgRae0zTrmRcg35dDwAPjSuki5BX", - "VlEp1QUCGsTxWPxvq3/8ahhNf/vOp1Dmo9HBffidieVv3/ksSnHmSYUJo7gr2vb4lyfgRZ1DoCzUdSsT", - "tpvjwDLRQHoeyf7fTkEqHcndNSRPhV81pE4aUmW5NmtIbi8+ropUr4Zx5zqSp7fQgjtM8T+nlvQnd4/U", - "NDidz2Y84kxATRFITNdr8YCoyX31jNwyIVk4f2QlmKgmiXRWIwuutUVCL8sY33kg2WlZt+mutUdfMfnL", - "TKySmStB6vS1UlpoV9g+N3oY3e3tdfeK2pdMYqgRrS9dZoXuQGUgrImV5piFVmKJQZQxUajWFC0OyUkB", - "IeEyyDTW1QINASvvLqyGEKrBVS+rFaqjBbWjONP9sYDKyvYxQuHsXbEVVs3iUhTJZcVMXfGpUDJZvWrZ", - "Jz1GH14IDZdk6ySE3vExdkU2P50Q+slYx52Ie6e12sU7xcEAjXvKipMsi4xw/paL+e4XFfGMzKqYWwX5", - "MCBq7dHcyIHLd91bSARBC+NAnic0AhhI+xoilDmIAYQkrDYFuAFKJglTiDyX5caX6BuLYnBcVEqQu3o4", - "l7b5SS4MTy77GE0D8CGaULFy8EtjUeuMGsPSzEAaMKS4wwgVy3DEjdqEdtBc5hregozdapeEJtd0pcfC", - "JQ7j51DHVbEIQRqTZEh+lIDZQOiccuFgBOybWNnvGz0WlzxO2MRBLlwSroleSGWYYDFJ5ZLper+MqoQz", - "BZM4oXblNEnpCrDPEAYS10dmDPHFasAO0v6biphD3TvbczHl47Gg5GA0IimjQrs0bU1nzH7l2iAwiNqA", - "viWUHI0eua8a+wb4vH75d+x5UYotZUSnyYowS8VwI5pd2MDifEEJWbt9M6407ldhXnQFxmoby7UvhRv3", - "SS7KRHQwteeiyBu322VyJWCezgnHuCoqHDrsjSmLqF1PIev9AOqhjKJchS5Hu9WVGpz/jkJmZXoXsFTh", - "NO8ENHYrm9g9F9Is4ExLOEq737ZQVUlUf45LJnhIpCKUVOi6NChAuXoxJzuAEnhZVvcTvkDs5e63/uzY", - "4+sYgT/+iNP3pdxPQERyNqsdwO1XEx7gTekV6yT8Zz2nJ76sa5XFxZzOhdSGR54ZNguPf1UeOyuPm1c2", - "SM0zqa6qslWdfn+Q6qqr9uWwR/mXpYRVZ/gZ+gHs8ADn+dO7A8AYjYqKJZo7V9Ca9FWcUhC6uNE+zliS", - "RIq5PUWlUfzOrfYNjS5KchDLvSrnjSBWCZm4H7EyrJ2Mq7sJFv7ItfqpeZHt/Q58Qb9IQ3iaJSxlUDl2", - "gMRmN7tEY4LC7lxXMIluxivtqaqmDqMuqNH93/fiENCV37AdkN7XtyvIVBM53475V3TuAe4CoH9jgYXn", - "ma9Sf0kKHmwFWkTYJ9cLHi0AABD0Vts+4gPSLLsssI93j8kzOMhVCGjofAdx9S2taZkwxPVbpunl8Xpt", - "1NdnZ/ARYv9hFdTLY+LroRb3h7ZvVQH97CwSqg35xcEU7hTKOOzopaFW3yzmt+ug/kps6rEIwf4Jdu0a", - "5DNyWUEAvGyBp/L89mc5/2TCWL+9ogDOxUjiVEegTSbiXluMBU/C4H/7o1EI6LojECEO4yPjEK4N5mc5", - "L6oZ1EiZZllX8nXDBCpepukGGiY7FVQzbWKZm79oEzOl4GNH3W3ETXZo5CpZ0StLqA7Dzh/sXSC/YCQR", - "wosHl8oy1V6/x0Se9o5/df9apmmv33PjqcCS30C43wLo2GxwPeLF7kwFtfGrWH4TPMY6s68AMjZuDqdO", - "t0vkL/CFP71n0dvsPiEZgnzQMOJ+TiJoZbx1g4+QBbDkl4Ue5UxNDamt3VTjZzmww4tzX26nSwjHhfv0", - "wn/5GWi/2yI7/JiJn+6dh3isj+BLzoTVa7OZSdWEHNoW+/HZE9KH25K1qXahkK+0eXM7XyfCtJL6Ootw", - "H8RYfo3mRqbU8AhK/0QLKXWF7At8YCzS5cy3BWWCcQP1TBdCf2lJ9dIZgi+dIH/sjFaEVh+5PobwuQu8", - "D3/hH5Vf/FDRywuO3/fCN8DjQ21yxdmMZDTXzMpVecpItIosV8RaT4xGCxLRzOSKQRk7RlIueJqnVeBn", - "u2NLCiAVl/vpZZ9Mc0MSquagF+FDDKdXLJJpykTMwEI2FgtGl9wqdYok1DARrQaaQfnbJSPXUl0lksag", - "5LsoHCyfp5ilQEDRTpmhMTUURI1Le+InmMVzWVTERcVasDclNcRjoXLxLUL622Yv/UAvCQPMaq4XReXE", - "iMZMREEs54vPm419eGvwBTPNiX6iuJxb8dJPGahTtXr64XweMTxfWDCyVG4bu7D5DUKvblci6+kPnoz+", - "PY80ztXP8RO5eIol3nSKPw/fTkF0n41/59M7cKQicY7dVU4lkPmf1StTMJRquBOkVuI23tY1U5SIK5b5", - "Rjxv7w//5+ktrGmfCSfstyr2bcWIykl/DizXreqteO4nMiM6W1LFKvYJWbCPqfpk4pNUFS73pZg7HcPG", - "o1nw7Sp3MoqC9iXFV7bdZNsu5OC2bNvbZtec6hVGzsUAojTDHNyZcVtZtTMd/JvmgjRmV2GZn5xFlr6D", - "O2OLpwUjRNaY0VUiafxnCNPd4MGJpFKI/wCIEl8S/mjFalgN0AfbXFnlrO9zK1+fne22cQllNvIIZb5g", - "DlFJirGfpYHS/8+XTCkeO5hOcnL2xAXMck1ULobkecoNMZJcMZaVOSUA5DG08/NIGOsV1WuQF/0eE0at", - "MsmF2TqK8tWPM5h3t6rDfsd80gFaf3VId3ZIg2X/y2NnwGUgawInsFkzNdRsLbTJxUyqFOUyOpW5bd3y", - "ILtMdj+xYOWMJ0yvtGEpxgXO8gSOGxRHcAVw3Xe4y32IirUnBxPWMqZSrjWXQo+Fy9bImLJ9289t+5UQ", - "p6BDwNCCv54jk/w8wufsYDBijJq2VQPMIiis2Tvu7dEs24upoS0hWm547zGkHyAejuhVOpUJj0jCxZUm", - "Owm/QvWELDVJ7B+7GwPqJvDdhy7ve/uTZVf6VMxksHga0mxBzH+qvCrH1rxj8otja89Y9bB4/gMbHWZr", - "2wsIK0aTARTk9cg1JDc84W+R1dlGuDY8wqSfErLg9VmJWjAWZ8wo+w6F5LIkYZHxBpu9TMlob5yPRodR", - "xgH+7JDB4IDhtT9OoceT81eYCMpSqVb9sbD/gIZfPj5H7+6MOmtCZaCucjA53Xu+JcT4Apbp3zhGDye4", - "ETsguOFfXYI3RwRpPUO65YjKbJOqJLM/fRCpk+C+2hW+TLsCQDIVs9mZKxqBUKwXuYnltQjbEJYyyVP7", - "D/zjdBuwl6HR4jW8+tlIuzicrd34CX4Rh9LNKWZY3PGTOD1wwb7UmFW7cH4KIMTUogGDt8Bj82ek7g9v", - "vq+u42fo7nQr6gunfjZn665vPjcGj3FRXY8v5ZgjpfmZGLnZ+nRNebv16ftERlfagaFUzYZWbwOAcftj", - "CQjtXIQgJkBuJnEgQoS9ybgC5LeGARIxdzShxDCVckGTPZgzNgLQ1t6KRZeSQ4p0lHBIUuMxoBYlgE53", - "vWCC2NmAoco3UPHoaldaqvpO1RlpJJmySKbMw33vhlS3v1FufpCqjt39ufDFl5X1t/OxU7Xz3AJX3t7j", - "e8GXn9E3ECod586h7Ee080yWP6IpqE9gb8a9w5Ee9/pk3DtIxz27AycUTKjUkHsk5SI3TA/JE7RvQRLs", - "/RHRLJIi1h513FvwDke6LSUWybIlv/I+fHeXYo+jKljKF66TEHuw7xH7PSTtkJ3qgXNnMu7DoYuJzA2a", - "+925cm/FzIB5ZPfOfbWVM/JVt+/Cyf/mjm+NR8EuW3ZZ2Xrk7FmuF6zd5PYzVvLJzRTQrH11Tb0gf5dT", - "3SeCXaM1XGkzXON79utz7OAukPZtVzdB2Xdz/wqx3wFiv1yrMFwiBljaK9lTB2ImsjeICEt92rujIdAk", - "ALtBRjQhz09OxyKyrAjB/RRLJXAnBwiOt/Djv12Qpycv+uQJVHokP+bT3SF5LpKVr7eNPpqxQEkMmVdE", - "BZki1bI4dD3j2IF6PmawuO3gE5VOxpMR8Kz4vfJB4v3egtEYJJI/ej9L7CyAEfziZ3uA7P65L4tt720U", - "PnovmFGrweOZYWq92TOXJyUK1Ap3SXsQOCe4IfSk7VB77LOyT5QNEJzi8KAXwKp497XqwcevEHo3XjKM", - "E8F6c9McsD4FJBnQePVlxTLpBSmYY4gFVq/rom5AW5aw42UbFQzosi3y+zMyuW/kXTUk+H/X0wUz/WId", - "TVltnywRF/VGtnp6fXLwAgGJnaMqohmNuFn1CU0Sd0e5m6CISBkU4u9UMXoVy2sxHIsXRaUTl9BLTs5f", - "9b2jlsRcX2ELzhc7JM+XTOl8WgyOwEFDrzGsOYvHwkgS0STKEytusNmMRZCLCwVMdIsvtxhK7yOenbKT", - "YLWVSlR7/sUVeQvTBOxeSRZNitvDrd5TLEooT9vhv52gBgGHEGowtY1KQbiYJS6kKlJSa+KaGrCEz/k0", - "cQFCekheLhjRNGVjkSVUCKZIrjEq3g59kCmmdY4J3rYBgMlFiuqTEtovU9K40IRESqUxmsBS+Oszog3L", - "NpDZC2z5DOb8kWRbbNz19ImM1I0xtJtC3CvEbghSCi64paM88QGMdxqKjgP61FLil3LwXyo+nzNlTwVF", - "JovheHis/XLioa9lLLcWfLwo3upW8LFotZKVWMnY2wjNNinRruPezaL+Ap1f8Vb0PvfoZlnEP9mPOvZd", - "z1YND8I9es9Z/lnq6F9UkgS7GrBKCv/SzEmVkdeOai3RdjusVufM2o+Z6doZP+uTwWZ9yWhZtJY+26bw", - "fn6EMLpblIe7Lon2ZdNWDe2qppu2pPxvx7P/LCjw4wDZf2KUk1sA2X9WefeANP7p8E+CB/VT5dHXfM++", - "2uyfHov+Y6XPIyA9wLG1pc8j13PBqxsVpdfunW5qkmvxzyTBu3jHG8jvftm/av0dVIbKYm1zQVuCZ2lm", - "Vj6gzfkqy6Azzd+yYYsjuIhb/Xiu4FuEdH448vB02hrQ+ecsDv9JYkZd8T6uyemTQNX1LwxjsHrmahfL", - "nr11BlRFC75k7Ub3+gl2S5QpNshkBs6VGBfMrYe/ywxVw/lb4pp3mKvuX1D9EcDyWUxirlhkkhVW4rQc", - "Afv4RhMlrSYAz6VatUeJ4BH5Qcn0sZvNlvvQnSlnDCvjDNPVIKaGDpae22wwob1HdKePp7QMj3BBnn1P", - "dtgbo7DGBJlZzYfwWbGkWG5fA03uVge8P2qxbPK3bDKfdhnlhmohz101FhLl2sjU7/3pE7ID1cfmTNi9", - "sKL+DCTZTMklj1lcG2NvKRNc1f2WBb2p3dUKFUXpOK9c4OA+iQzT5UKav+VZnS0UITFTLigMbmtdjvqZ", - "wiR+2x/lwgfguD3yo/h6hTnNb8crO5YSoRKmW0QjJUI873695r7ka66aDOXvtNpt58NzNhuvu+VHdUxb", - "+hiFH4rcubs1W7/+fFJ6uP4is3mc6XxZKKRtZvPPiwRHd3c/3LW5/PUXnAL6jHnlu2IqhwZsiyGC+Rli", - "umO2ZInMUqhIDu/2+r1cJb3j3sKY7HhvD2K/F1Kb46NHDw5773579/8HAAD//zZRKNvx6gEA", + "A5do2QeZ3WgbDrYYEbaO5paAsHcBAtuUVCoS4geHfK264HyWMFJoB1dcqUgGMyu4wK2xdLJhTqG0kUkZ", + "LFhtVa/0zGEeOumZehEIsYkLSITS0a1Xem+a670s4nsuJ2UP8CoeAl7FUTChOGbLSZ6HtCH7yAOUvHp1", + "+oTswC+AtwpphXWapfT+/sPRw0eDh9P9+4OjeLQ/oPuH9wcH9+hodhg9ONw/ONyQHNIhw+z2SWNBJTkQ", + "21tEck98RHko0Lctnr8hjrgY5WsuYnldu/GCQaPV3l1A6rbu18PNOw8hmKSSUG3QYtHCvM7gXmeRbRuD", + "tF0Wo7eMthj/7r8c7W8x/m1lYzC4lmvhpcoFuhox774wq6eVAVc3qz7O23F5GJBPBtm2WtXOuy/a6Pje", + "o+N777toPqFh2xib5HSHm9sWJeVxiRsZEz5rr2I68jiePScGoaXbJVj0+r0iBwT+hvu/EV9cPO6U2NR2", + "YPthNrLpWmlJKD6tqSgQvYG4dPGxFU68CgLo/0XaupV6ThKZx6RifkOYLvBNnVbUFdsMwWL7YJVD2E1M", + "ULBqDeA8Q5UBLiwjBp+cbcRlHx+TZ/AuPKIpanJuEFjrouqOovEKY0js+fJdo161ecgXTqWCb6x+Rey/", + "YNp2GZyVdnMTKJAdk18kfFMoeEI2zb34OmhW6683TcM7DsLao0lAZ066PCY/FBJlIZM6GXRHM/fnxDGs", + "EsRlt5ZK73a8Z6ml3LlKWni/hyva6/f8QkH6+Hoi+auS6tfOX5UUQ8FVjCZwlsu82dxwX/8eZsK14ZF2", + "CRV2c9vEHleGh8UT1Jfa4jQxGdPpVMVHXqp6fUZ2AJ3wL8TZsu2/douYztpdd/Do6NH9BweP7nfCICoH", + "uF0aPoFU4fXBbRWNoyyfOHNI29RPzl+huSNCQ0IRD/L6rAr5kClpWY+duW+w2vmj4aMq9FIscyyd74bk", + "cNoQ6RU2LIguVvCiltjA33my5LOZ+P1tdHXwd8XT/Tf39cF0vwVSFjsKW9pOqw73NbM0mw6wpE4YHQcI", + "SulWAKkXTMMMyAUzBOhnQGgEGk2R4etIzsNMuRUPEtbR4eHhwwf3DjrRlRtd5eBMwO4WuJTdCCpHDN4k", + "Oy8uLsheheCwTY+zAKjfwmm74XNGXFnZUV0gHe6PDkNU0nJxl1Tj2l6mrUv+2mmMblJu0SFRudAm1055", + "cLUPD0cPju49vNftGDuL8ES92cxhfBoPLo8Dp6/u/A5Iky8fnxNIkp3RqG7O8VFTNxqVudGooLACAqLf", + "YGAPH9y/d3R4sN8NCS0UiOEw/moHts67AocuQBSB3QgsxTrr7bfdFiFxCgnsBYsSytPHkU87aNw+CHw+", + "UfhauQldLganga9dXB2+7WTPKqxUmLSCooFUJBdFuY3hdi/sB3GmtnNtvB62c/VQqoqwq+cge7Dc1i2W", + "MlNsyWWuP0BD0mAe6SyRUt3o2zaF5QXTeWLQZsM1eX32DfAUS2tEG5bVdShHjRuAjW45uRud5xqJhIm8", + "bbE67UaXrd804X7Lqe1vApmocYNWOLHYcq5cbA+IPKFJlEOBGVrsp50V4GJBmnyWJSuMd08SKQWJFlSA", + "X0RVUIDIQibxMBgdap9MZsFACnlNEolAyFeMZa72Cg7CfmZFGL5kZKeSpk+QlBo1Mu+lyGRcdY06Nd5L", + "w8X+qA4lcBXp4XY9qZEVjF78pGYJTeRcg1JoIJJ/2ISGz6jCAH0qsJbQMkVdMhCEHBhig5mHblS8SeXM", + "KbhO5IDka1xJGimpNWEJn0PdmtdnjZzeDXlgRWbv9iDH+mA7kC76MANXGSJEdS45FrofAzku73NDAg1D", + "Ht2G8EFvnEypyKEaS4WQ2ZuMKySPbiGCC6nNpMBquuFgtZlAiYVcsRJBrshEL+xB/p3gvehZ222Wy8Xi", + "3urrNaoKN9U2wHaeGlzR8Gr1CxoMkfE6WtVGgKwScasJr3QTALcSk59raJVXoLzIDuSBVNhSBZZtt0tc", + "TFhltf2saauukuTPR6OLrlBnm5HNzqlZnIqZDOBh3MBZ6i3RLoIzYyrlUGSExExwFntdsvCaOlMXJFEn", + "mpE4Z27lUD5V1C04xeMNPivhbWRczBu8vtlhF/MwjmFzBQbo173YJcJJh5NMX6oc1gpDFDWhZbppp7hP", + "ridhd9Z6w4rN84Qq4kAKuwxZr9KEi6suretVOpUJj4j9oOkKn8kkkdcT+0h/B3PZ7TQ7+8GkrVzPBQ7O", + "5crhhjT6LafwnZ3lbiNTFywxe/j9HjhGuwSMBaOnf+AJc4h3rwR/UyH0OkT50cGoLYO8pdFa7vg6WuJN", + "Obcj2eCJz3Ug326jlOMLDbHY4caj2JPlGmuedLiVPDCpdwHezqNTT6Z4P7iOE+TXDbAOMmWQC+Onts41", + "OrDFLlMJllvI9YL8XU7rBtGukb6BIl47ooSNUGwWjHmHHd1okMY31taksrs3wWUAtmonCh/dEO5gW7mz", + "MqSqjZ+8WKv8tWBuybifI1YB61DVwod8FCn9rtfuuf3NGm2BEGVAbtFmBdVOocTMqlJ4UJOpVApQma2E", + "I4WfDUCRWJnHrrXHgiIvF2xFFEspF2PBRWEkBYAxRgRbMlXJHJXKKllzFg/J3yoqHuBYp5lZOYB0MJ5/", + "o4m8FsUYx6I6SNt4rm07jwVaFlUOheXLl2yzoPVZQoFMXnCCGQU1BblZkJlielGde6iOpJXxrqWKWwv0", + "rIh/Beq+gI+VGHnFRJWVFc0EVUNsaIJfrQfuYRFYeOr0T1Kry0qadVc395drpsJCYjGl4pVOoSuVo1JR", + "ThAYBVBCoN6e+wtZfIEM0gEHpGz+r77J8qfzovH6b43XKlgfHnr3MZptgybYCDN3GsE+dU/a1lAVSA3b", + "hPCy7ksgOz5q2hctqUsCleIhne7JbtlpzfwAP5o9zaJ670cP7z2437F6y3s56xDR6kO75pbpBpdcy06d", + "dfH7PLz38NGjw6N7jw5u5GHxqSQt+9OWTlLdH7LD3hh7WJN//eOfr88aXp97EHY9utGgMJkkPKSWhJL6", + "gF6f/esf//SjuvWAQoxmHTW7xW/fGqWTVHfSBwrUXXjdnGQb9PvHNSMBLdgM2WGzGQMz6ATXbVAOpgGN", + "0U0KphmNuFkFGDm9xgD34pUG+nMXd1B9sCGRF9t2SKGWc+l8WiZi7vjOyX+ib7hBCw87F4HS+bTND/28", + "2St6oUuvRTXGoUOIgS7qj68buIv5XFNdi+G2f0eQauGTytYTbPCNzUi0zewHiGJxtc4qoYAhBPOGPOk+", + "qm5/YzsrfsuaWae54r9tOIftR/BGVt/AjRww+kbbs2kb/MFdgLf7ajKtlmfbWP+uVsutvHVv3m+HhOH1", + "2gHFDXbz/io5kjf5sImTC/ToxuCWvGy7XyOJFmqqpL8EDGgyYYMiUM/lxhCdo0fQnnkHvR5I2oyu5GxW", + "x3+9144XDlA4kN/le6HGWM2kT9gbb7Nogk0j7s24d0+Pe1YFGPf203Gv4bYKZkym9M3EdVDHOxltAvAu", + "CpM2B6n9DKaJjK6w8hgUtB6SEUkZFZrkAg5/w6u2P9rsHer3ssreFHDZDEOc1tgWjGnKFnTJoUqD86nM", + "a4GY7A03GgJGoZ1jEktEQKqVXXUztK9hPuNxOWm4dKhYuYZtg/Y9KXxEa/kuGPhmUOxVvGVK9l0Cv+XY", + "z5+f9TGAAUIPcWC1+EY/URyBZZBFF42SA+Xv4fjhacImMO4mhH26vo7VNHTwrCqmmdEO07okhwYRkEjm", + "wjSx7dNuilw9k2z9SsoFBPu58A/AKnO9u5r9MYvgROr1s1gn9FsQdyNvwK10KHHgMETCcCjAlxT2Fb9w", + "DuHmANDYUKmYjO1U47rRSzjRRroSW8WpnrA3EWNxEwQz/ErXWHn3ZTBW/mfqcHOKYsbubYh3Xp/d8OPl", + "dMFY21a7GtMvpBgAYoffUoeugXB5Dr+lTmg1OO4KisUkBDkaeqFLkjV7s3mtf2FvDGCGx3mCQHBh0nWs", + "yl1G21b81smMbQdaqu21/z9CKTeMN79VMTcXqv4p6rm5tz5KDbe13blgxr974ciovZJ/rfhJzaXlA/79", + "K/UYGySlPnEXPNlPdxskeLQIW0UcUG3HtExBUzbJFJvxNxuIB19AxbiOZFIepCKDATE3d1L6hhw9INGC", + "Kt0Yu+DzhUlW9QCcowC+0HsVOlTMMOENhV12vtxN/+F6tJvbzmrrIeH4ooIGtFbmw4mkk01Y0ielt81Z", + "5zO6AitOq5PwweHRaHR4MLoVmLQf1g2W66T8xJUFrLfTllJX+c45+mtRqtUWirzq9Vqz14pDenaxTNoo", + "RtNjSLzJaMRIwmYAHFcktG73LDa73jx4J1C5LNqC/v1GuX3zPvh6GZmiK4fD7afR887FOuxQ9fkWh2gL", + "m4nWYOYCOXeHg9H9l/uHx/fuH+/vfwwA6GKR2rI9Hrzdv36QHNDZUfJw9eD3/cWD+UF6GNTDrjhWy+lC", + "qz/Zd1ujbMpLsg5fVGNpZMfNIWOqWUS4WXxbs4QLNtBFhtT2NMUNvAD971vP/83s/DiDjbLDRX2SVRGC", + "mnJxapT1aSC33GQ2+i6aszl9snkWt8pAag4kTG/NoQB5dRsMVG3Y770nGEMuOl5Dryovdr6INmbFbbuK", + "Qh52OOnBXW5Z8RB51/hk7cBtusDXL7mA7XQuFTeLdPNtUbxWQGtD3PRbbeI6xNOQnM4FVBCv/lyEyVWV", + "KPtxr99L3h7Vz4z7vTvYl0PlLQjQbXVVKugQRgYF6jevArxSKh4KI9mtrm7H/N3+YP8RxCEkb4++Gw0e", + "1SMO+rha1eXb92/Xfh11WcNqWTxfTmn/0Y0irv16bqKgn3ioqFt5Lzu8XkfjZb1mf3X4hNvaBpeP1/a4", + "Ad7TKoC+r6TnLrdJVWiKWUJXIbz2iqFWN7THKpGRKZtzobvYbQ9HheH2XjruDcljB5oNumxZnb/WPNRl", + "r9AJT1MWcytjourfnsFw0NEW19Qlblavw38VkNaGYXHt0XaIhG0JV9uuyeF75OO+l/bbTePdhN4BdjWv", + "ogJsGLzYJ3xGqGgU7eRiSRMeu0R6SIyEeLVjj81WkqzjAbqUA72dpE/m0pAyhb6jvS0X7XbBYvzsDdhb", + "N2BmIEEcfBBAlAIzjG9iX6dPSKZknEdl/mgCgy4RP1TeQGXbIORvD8n9mPYNSMyeSUW22zfaDBrd7JNt", + "+92wTVqCbd/q/dH2rf4oRpF+L8/i7TwMX+rGwW6EZr4lBTFgoqkve0MSrEzmtw4c/UV1Bdd1XrQlR1Yk", + "yjPvYLE0tU5JAXcLuBhCcb1PWMLsNbXeCJFJXGZJcF1y0e0sdf/+w0WbixM8UusD+YmxzOoqgH8E/aVU", + "rIID86U4i7tkZ+TL7Gp0eA2whI9brfrgHmyVxFq3qmrCbSsrgFy+YfNGvJSrivm7gPWuimbrCCie4deE", + "tBftsPjuSx/21mo//hhmuU8ppD13rocGUqvHES4A0X3/ZSywFevqxHsUcs+HyOKl04zb0GebWaBVq/Pj", + "wf+glZlMhsd73/3l/x789p9Ba3NDb9ZMDWI2g0CjK7YaYEEeq6MP69irUA3ACtNzRyqMpmBDAlxzdxir", + "4703KpjG6heark0BIrQq1XT2t07oL//RHt9UWcZXwCe3kux7F8v4GEVFjfTX0U7K1NzHkvtEst3hWEBu", + "2hVbaVKp0eVEGk+o3+jik0oEOrlEMXDIxPKSTDkUPdRjYbVaGkUss9qEK/vCsXK3BO6jGE2q7bhaYT7x", + "2zkkMZ6Akddna8C9z1+9/P75q1+eTJ6fP/3l8enkp6f/DSEe1wPsIR5Y2ju6d9/V666u5H5gi9+jtMB7", + "YeeGyA0ROAP0BUmWUAQ9IDBzDRAJPtyg8jLZgWQFXwDU52ru3gwR9HHRYDDY7QPXeRk9+hC1GF9tLL64", + "lMnAStQtePxBAyauRTBYG5rCIPhem517Pg2Ijc6aOOdzGjBth0zYH6Rmoh/Q1iSdtf1vLR4VDp1/0ixW", + "gNwAl6oBrt/QS7UZtEfWp1aQasF6BfxaTJfklXCuem5kKsyeq20agmiIJUBmb0qQLU+ZB+QbwEfb8z43", + "ivKVmVVG0r43Z15ibcjUGxbo3C7N9YIpVtkI+KAEgb/hkrlUkA7AH1gbLmOqDJP0eSRWEAIPpyY7hbHB", + "L0GR4Lpugd0M8n9G3xQ9gPWe6jWXF8yjrMKz/+x7AGR/4SsP8plvAobR0CfC8ON1Ktq0Jp6q1jejSlXr", + "88b3gwfP8aoN3K/tbDWIs+yjRpohevwb5eYHqUADaYfZ+Ogo5qDdxEwB7FgTo7wTwDdPWTyRudl8/l1h", + "c4exUVSnLKubem2LAhFHtQzSNl7ggSDKMayvtF0OFuWKm5VVz50wPIXMO19SFBYSOoKfy46hjOO7d2Cn", + "nAVyFJ4xwRSPoEimPY8pFSCkk9dnlVppWDZvDSIUxMvnJ6dOw/Uos6CxcAOk50P9Hp+f9vq9JVOo5fVG", + "w8PhCA5zxgTNeO+4dzjcH456IMgvYIp7UNjcpey6FNdCVzqNnST0vX/Jfqloygx88Wsg+RxC3dzrIPXS", + "eUVvyShXTnHJEkiKR4Lh9msAtfcX6jHeyn1c9s5mOshshYQLlj13m/sbCJVwdmCaB6ORw9I27vqFdBGM", + "Ud/7uwtYLPvtJNW5JQpgvK9pFl62LJb+Xb93NNq/0Zg2DQXObqjjV4K6vFEGCuG9Gy7ErTo9FZgJ5vJ6", + "XQRO9cQBIVXP2q+/2T3TeZpStfILVl2tTOo2wZhpQv27CPFrNIksq4BKNEPyXDBXzZ8aQjFYVuUCKtz6", + "Dy2F1k8Btu03ucDF+V7Gqw+2hLU+vFr8rs7O7HF5t0bPH452CjJe30j3yIM6I9XeAQF9T4vyzJ/spByN", + "Hn38Tk+kmCU8MmRQELALgeUaokwSgKj2cDdSkd9zaSgpIsi/oCPtZNZpQW798ira+4PH7/B4JyxkeT1n", + "KqUC4/HxnS2Hfu04oxW8PM4bbzVP+KdPeu6m8rgveFGBIFc/otVrqykMrl9HR4Gkf9cnTi/+hIR/dAcn", + "3E22qND5KY8clH0kuWZf0nFyXp1pKYQEZblnzHwuND+6yyvL4db/CU/Rl0LAz1gh4ZW7tXYp7GUqF6gA", + "ByXAF2WOnPvum7rw97J8UgnMAFO6bRoqKBh05dF4NSR+TVHpNytA9VEM5hmvXyvndnifywk7uIsTBjMu", + "nBNfr6mv19SmU47U4qcAB7NyyjvYIG5kgfjz2R9ubH34anvobnvoZHkQ7NpZF/4up0PigiChPL5eyDyJ", + "yZQRhNjx4Q6GquH8LaEqWvAlAxw1qAuWJ4ZnVEEwQ0piaii6bVsNExvNEkVze7a5gQ99Kxe4CZ2g2QSg", + "3yZtkIdl0BsXgsXEfuLQ4koEu7Wi1Xj2gwb2osHyaiTXC6lZASEnTOU2h4xajdoxNDsci5cOW9QuIMTv", + "el6jWQIIqRvsP1IQOhbug289C/GxR5qmJeeiCmDqOIIh4rasZ1PZkU50JEPwLi+ZoMIMdMYiPuORm9YV", + "W7kQwmCDnUr92AH7cb4+K3IEyMFuGCIMEAHDeLBPimfEUVLdfyMg7jZK8rh0cnnUGqqmNEmCtSDmiZzS", + "ZILrc8UCPsFn8IZblGpxe+9NEjJmWKg8W5mFFPh3Ps2FyfHvqZLXmqlxb3c4FhD779aaxf1SQCTXUDss", + "zaQ9Z0qm2OceDnHvjyu2ejcci8dxyoWnCPiEJloS9ga+g5JKANOA3KuFHvA0hf3gJ7k2Mq2CbXq6w2HK", + "3GS5cUkMmpl+CGhyLIwkf3g4wXd7f5Q9vgNnMaOxpZPKKzglkK3bRq0n1M5+Aq8G3O0MFmDcsxfpuGf/", + "nisqDCJFFniIZF7d0p0CkB+KdDZXOKKCZDLDYgZAVAtqSa7WBsAD0CQhBo6S/9YK7rCTLfNxaG/ptBXq", + "DbG5GseIC3L2feUwjY4ehs+TZpFioYiS/7p4/guBW9nuAb5WRghhFoGwAgOJc3Cdep72lEYLgo4qqF83", + "7vF43CvcufEujDXXLmF+MACf4nd2aN9hN30efzcc2qbQXXlMfv0DWzm2ZylLEXpy3HvXJ5UHc24W+bR4", + "9lt4QdsQsy5qjIDs4DW3C5yEcgA3qdz4eEVSERPpboFkRSgpOVA1cGXKBVWrTblrgaV3KyhnGDxXWYw/", + "xhAsN+4dj3243LjXH/eYWMJvLqZu3HsXXgHntWwvlgb3WeHcLIjo/mi0ux182a1vwGfZwTHwgXXAVq2o", + "qPRod9Ahf/65/AP/1vpn4fqhwnZeouGg4u+N71+gA6IisVc10YALoiF2UxGxxIvd2w09d+88sJsVsSS5", + "awL9VORZuMcKcPgvihxhs8pjtNF8/4kpbnRXl0rNbP9p6PeLs58HrOfOds6WPtQ5XBoDYE+cKk3wZUI1", + "uYAxDS6s8v0Ufh26/3rdD2D8LhM5vzxG1Z0kck4SLlwIeiVQ2YoHbi3hI0Q+Kb5zQCi+LtkOShL/+sc/", + "YVBczP/1j386OPF//eOfcNz3ENELyhpfLhhVZsqouTwmPzGWDWjCl8xPBgqPsiVTK3I4cjZ/eEQq1dWd", + "lKbHYixeMJMrUQnVxxJh2jXoXAV2PlzkTDvkGPsin7n6JRjbGLDb+LOMS3mnJ7ofQOCDGVQmYG9FTwMA", + "X8axtrPTRHthkynOuWY0bYZprgXrbecvhr0xSL0DHOANGQwscejcwQM3abJzcfF0d0hA20KqgBo1oDuU", + "zTg1YviVJ23nSchR6gwFVhl5U0QzOuUJ9ybHlgIbeARTGi24YGV8cQFv7Zs49iO1PObx+SlxgZB9eHUs", + "nl/sgYnVsMjkivUdJ1AO1LKswCVd+iP0APyLG4gOG7h3x2LGKGAtnD5BJlDBfS5S0IqGBWBHQIwrFqwv", + "uuqPBYKXOrBce/BSGbMEPoL+59Swa7rqk6K8qi/IkVBjFWLdty+PBcKLujUYADoGqQxzCPwMhzTwkbwu", + "TUixWWJVY4jAx0rT0PfOTCriIpwrheV9d5jXh8Oyi5bS6PmFnd8cNEGJ9kBo6fmF343dPtGSRAkHaoio", + "GIs5BAJ5vFgparta5DAtqIoHkbSXQBU/6ErI64TF8zYee1Ilso8oydT6CRynH5vk+qUJF4v1CdhDjJho", + "mz13T9w73Vx3rsU/k+/O1R68gfMOLbgM+Q2u7ldHXgdHXnjdvFMv5Fl74kH/Pl7EL3bxiQJ+Pe2trzk+", + "qSzZp7DokR2PpgJeEanI+ckpoXGsmNa7/972PjtTpNJS/rP3o2XFnyL0xI1FKocz5+wtdQL5UtjBCzdq", + "Qv28miVdq/fbXq3eS+tNV5R+Ka+8j397NDq9yTVSCr0lrX29SbYG23IdSahsV1LLAESjhBXiS3FOq1S0", + "zaqMYbzFlbNRXHLs+fSJP5B3Z192XeeieTfcAVN80mCIn5AR1lOtq4WavyRqflXsogc43mB+/rxIc3R3", + "UtBdm6JDZP4lqYtxY9ksF0RQkNYL9BkzCAXyMfV010Ng4hdM+VPt6+rDrItp4acEMU1gQmCJ2az7nuIr", + "3VRfbO/PpPnC8txEYnFL/lVE6aDslmu1ScE9dVWHP55+Cz3cSL39cGErjsACiwxW1Kl3O4FldYfqlYh2", + "v0aufHCKxrjGUolVft4sLizZiKZU6Fl3Jdc9FtUS11amc3otF2SW8PnCOQFiPoNYPVMtGQ2jPLiDURal", + "mRU1zIUofol5v+d2kZ0XeMmUIc9PTnH9q1fq3h8QtLpdVfLMa+Pt+urFzwMmIhkXzpN2mdQ9+cAKE9J/", + "LZf37k/dF5jPyr140CYwvsf+YzA5wfj3IZf/6+CHhE8VVav/dfADTTIu2P86fJxQw7TZ/WjEMrqrm+6u", + "FZgvmPis/sLriwasScwBpXSLwF+81VHm9+//qcR+nPSNBP9iXb/K/l1k/+pybRT/3VZ8VAUA+/hEHq6C", + "2EKrDY++QtrcgdHUUWQF0qbmRSpBbRZSG3j05eU3u6ByXlBc9droaP0vD+TG68OT7umTPiwkFDGGIgou", + "ffCOfAF+HHcu3Lp+794R8Did8nkuc13NTEypiRZMu6zdhNUZ8JcmdpfXc6vg/RlT6egur447l6u/0v1H", + "kvibG4rMGx1622R+/1ZXmd+9b2V+RDR1mc2u0kPfVwHabQm09pimXcm4Bv26HgAeGldIFyGvrKJSqgsE", + "NIjjsfjfVv/41TCa/vadT6HMR6OD+/A7E8vfvvNZlOLMkwoTRnFXtO3xL0/AizqHQFmo61YmbDfHgWWi", + "gfQ8kv2/nYJUOpK7a0ieCr9qSJ00pMpybdaQ3F58XBWpXg3jznUkT2+hBXeY4n9OLelP7h6paXA6n814", + "xJmAmiKQmK7X4gFRk/vqGbllQrJw/shKMFFNEumsRhZca4uEXpYxvvNAstOybtNda4++YvKXmVglM1eC", + "1OlrpbTQrrB9bvQwutvb6+4VtS+ZxFAjWl+6zArdgcpAWBMrzTELrcQSgyhjolCtKVockpMCQsJlkGms", + "qwUaAlbeXVgNIVSDq15WK1RHC2pHcab7YwGVle1jhMLZu2IrrJrFpSiSy4qZuuJToWSyetWyT3qMPrwQ", + "Gi7J1kkIveNj7Ipsfjoh9JOxjjsR905rtYt3ioMBGveUFSdZFhnh/C0X890vKuIZmVUxtwryYUDU2qO5", + "kQOX77q3kAiCFsaBPE9oBDCQ9jVEKHMQAwhJWG0KcAOUTBKmEHkuy40v0TcWxeC4qJQgd/VwLm3zk1wY", + "nlz2MZoG4EM0oWLl4JfGotYZNYalmYE0YEhxhxEqluGIG7UJ7aC5zDW8BRm71S4JTa7pSo+FSxzGz6GO", + "q2IRgjQmyZD8KAGzgdA55cLBCNg3sbLfN3osLnmcsImDXLgkXBO9kMowwWKSyiXT9X4ZVQlnCiZxQu3K", + "aZLSFWCfIQwkro/MGOKL1YAdpP03FTGHune252LKx2NBycFoRFJGhXZp2prOmP3KtUFgELUBfUsoORo9", + "cl819g3wef3y79jzohRbyohOkxVhlorhRjS7sIHF+YISsnb7Zlxp3K/CvOgKjNU2lmtfCjfuk1yUiehg", + "as9FkTdut8vkSsA8nROOcVVUOHTYG1MWUbueQtb7AdRDGUW5Cl2OdqsrNTj/HYXMyvQuYKnCad4JaOxW", + "NrF7LqRZwJmWcJR2v22hqpKo/hyXTPCQSEUoqdB1aVCAcvViTnYAJfCyrO4nfIHYy91v/dmxx9cxAn/8", + "EafvS7mfgIjkbFY7gNuvJjzAm9Ir1kn4z3pOT3xZ1yqLizmdC6kNjzwzbBYe/6o8dlYeN69skJpnUl1V", + "Zas6/f4g1VVX7cthj/IvSwmrzvAz9APY4QHO86d3B4AxGhUVSzR3rqA16as4pSB0caN9nLEkiRRze4pK", + "o/idW+0bGl2U5CCWe1XOG0GsEjJxP2JlWDsZV3cTLPyRa/VT8yLb+x34gn6RhvA0S1jKoHLsAInNbnaJ", + "xgSF3bmuYBLdjFfaU1VNHUZdUKP7v+/FIaArv2E7IL2vb1eQqSZyvh3zr+jcA9wFQP/GAgvPM1+l/pIU", + "PNgKtIiwT64XPFoAACDorbZ9xAekWXZZYB/vHpNncJCrENDQ+Q7i6lta0zJhiOu3TNPL4/XaqK/PzuAj", + "xP7DKqiXx8TXQy3uD23fqgL62VkkVBvyi4Mp3CmUcdjRS0OtvlnMb9dB/ZXY1GMRgv0T7No1yGfksoIA", + "eNkCT+X57c9y/smEsX57RQGci5HEqY5Am0zEvbYYC56Ewf/2R6MQ0HVHIEIcxkfGIVwbzM9yXlQzqJEy", + "zbKu5OuGCVS8TNMNNEx2Kqhm2sQyN3/RJmZKwceOutuIm+zQyFWyoleWUB2GnT/Yu0B+wUgihBcPLpVl", + "qr1+j4k87R3/6v61TNNev+fGU4Elv4FwvwXQsdngesSL3ZkKauNXsfwmeIx1Zl8BZGzcHE6dbpfIX+AL", + "f3rPorfZfUIyBPmgYcT9nETQynjrBh8hC2DJLws9ypmaGlJbu6nGz3JghxfnvtxOlxCOC/fphf/yM9B+", + "t0V2+DETP907D/FYH8GXnAmr12Yzk6oJObQt9uOzJ6QPtyVrU+1CIV9p8+Z2vk6EaSX1dRbhPoix/BrN", + "jUyp4RGU/okWUuoK2Rf4wFiky5lvC8oE4wbqmS6E/tKS6qUzBF86Qf7YGa0IrT5yfQzhcxd4H/7CPyq/", + "+KGilxccv++Fb4DHh9rkirMZyWiumZWr8pSRaBVZroi1nhiNFiSimckVgzJ2jKRc8DRPq8DPdseWFEAq", + "LvfTyz6Z5oYkVM1BL8KHGE6vWCTTlImYgYVsLBaMLrlV6hRJqGEiWg00g/K3S0aupbpKJI1ByXdROFg+", + "TzFLgYCinTJDY2ooiBqX9sRPMIvnsqiIi4q1YG9KaojHQuXiW4T0t81e+oFeEgaY1VwvisqJEY2ZiIJY", + "zhefNxv78NbgC2aaE/1EcTm34qWfMlCnavX0w/k8Yni+sGBkqdw2dmHzG4Re3a5E1tMfPBn9ex5pnKuf", + "4ydy8RRLvOkUfx6+nYLoPhv/zqd34EhF4hy7q5xKIPM/q1emYCjVcCdIrcRtvK1rpigRVyzzjXje3h/+", + "z9NbWNM+E07Yb1Xs24oRlZP+HFiuW9Vb8dxPZEZ0tqSKVewTsmAfU/XJxCepKlzuSzF3OoaNR7Pg21Xu", + "ZBQF7UuKr2y7ybZdyMFt2ba3za451SuMnIsBRGmGObgz47ayamc6+DfNBWnMrsIyPzmLLH0Hd8YWTwtG", + "iKwxo6tE0vjPEKa7wYMTSaUQ/wEQJb4k/NGK1bAaoA+2ubLKWd/nVr4+O9tt4xLKbOQRynzBHKKSFGM/", + "SwOl/58vmVI8djCd5OTsiQuY5ZqoXAzJ85QbYiS5Yiwrc0oAyGNo5+eRMNYrqtcgL/o9JoxaZZILs3UU", + "5asfZzDvblWH/Y75pAO0/uqQ7uyQBsv+l8fOgMtA1gROYLNmaqjZWmiTi5lUKcpldCpz27rlQXaZ7H5i", + "wcoZT5heacNSjAuc5QkcNyiO4Arguu9wl/sQFWtPDiasZUylXGsuhR4Ll62RMWX7tp/b9ishTkGHgKEF", + "fz1HJvl5hM/ZwWDEGDVtqwaYRVBYs3fc26NZthdTQ1tCtNzw3mNIP0A8HNGrdCoTHpGEiytNdhJ+heoJ", + "WWqS2D92NwbUTeC7D13e9/Yny670qZjJYPE0pNmCmP9UeVWOrXnH5BfH1p6x6mHx/Ac2OszWthcQVowm", + "AyjI65FrSG54wt8iq7ONcG14hEk/JWTB67MStWAszphR9h0KyWVJwiLjDTZ7mZLR3jgfjQ6jjAP82SGD", + "wQHDa3+cQo8n568wEZSlUq36Y2H/AQ2/fHyO3t0ZddaEykBd5WByuvd8S4jxBSzTv3GMHk5wI3ZAcMO/", + "ugRvjgjSeoZ0yxGV2SZVSWZ/+iBSJ8F9tSt8mXYFgGQqZrMzVzQCoVgvchPLaxG2ISxlkqf2H/jH6TZg", + "L0OjxWt49bORdnE4W7vxE/wiDqWbU8ywuOMncXrggn2pMat24fwUQIipRQMGb4HH5s9I3R/efF9dx8/Q", + "3elW1BdO/WzO1l3ffG4MHuOiuh5fyjFHSvMzMXKz9ema8nbr0/eJjK60A0Opmg2t3gYA4/bHEhDauQhB", + "TIDcTOJAhAh7k3EFyG8NAyRi7mhCiWEq5YImezBnbASgrb0Viy4lhxTpKOGQpMZjQC1KAJ3uesEEsbMB", + "Q5VvoOLR1a60VPWdqjPSSDJlkUyZh/veDaluf6Pc/CBVHbv7c+GLLyvrb+djp2rnuQWuvL3H94IvP6Nv", + "IFQ6zp1D2Y9o55ksf0RTUJ/A3ox7hyM97vXJuHeQjnt2B04omFCpIfdIykVumB6SJ2jfgiTY+yOiWSRF", + "rD3quLfgHY50W0oskmVLfuV9+O4uxR5HVbCUL1wnIfZg3yP2e0jaITvVA+fOZNyHQxcTmRs097tz5d6K", + "mQHzyO6d+2orZ+Srbt+Fk//NHd8aj4JdtuyysvXI2bNcL1i7ye1nrOSTmymgWfvqmnpB/i6nuk8Eu0Zr", + "uNJmuMb37Nfn2MFdIO3brm6Csu/m/hVivwPEfrlWYbhEDLC0V7KnDsRMZG8QEZb6tHdHQ6BJAHaDjGhC", + "np+cjkVkWRGC+ymWSuBODhAcb+HHf7sgT09e9MkTqPRIfsynu0PyXCQrX28bfTRjgZIYMq+ICjJFqmVx", + "6HrGsQP1fMxgcdvBJyqdjCcj4Fnxe+WDxPu9BaMxSCR/9H6W2FkAI/jFz/YA2f1zXxbb3tsofPReMKNW", + "g8czw9R6s2cuT0oUqBXukvYgcE5wQ+hJ26H22GdlnygbIDjF4UEvgFXx7mvVg49fIfRuvGQYJ4L15qY5", + "YH0KSDKg8erLimXSC1IwxxALrF7XRd2Atixhx8s2KhjQZVvk92dkct/Iu2pI8P+upwtm+sU6mrLaPlki", + "LuqNbPX0+uTgBQISO0dVRDMacbPqE5ok7o5yN0ERkTIoxN+pYvQqltdiOBYvikonLqGXnJy/6ntHLYm5", + "vsIWnC92SJ4vmdL5tBgcgYOGXmNYcxaPhZEkokmUJ1bcYLMZiyAXFwqY6BZfbjGU3kc8O2UnwWorlaj2", + "/Isr8hamCdi9kiyaFLeHW72nWJRQnrbDfztBDQIOIdRgahuVgnAxS1xIVaSk1sQ1NWAJn/Np4gKE9JC8", + "XDCiacrGIkuoEEyRXGNUvB36IFNM6xwTvG0DAJOLFNUnJbRfpqRxoQmJlEpjNIGl8NdnRBuWbSCzF9jy", + "Gcz5I8m22Ljr6RMZqRtjaDeFuFeI3RCkFFxwS0d54gMY7zQUHQf0qaXEL+Xgv1R8PmfKngqKTBbD8fBY", + "++XEQ1/LWG4t+HhRvNWt4GPRaiUrsZKxtxGabVKiXce9m0X9BTq/4q3ofe7RzbKIf7Ifdey7nq0aHoR7", + "9J6z/LPU0b+oJAl2NWCVFP6lmZMqI68d1Vqi7XZYrc6ZtR8z07UzftYng836ktGyaC19tk3h/fwIYXS3", + "KA93XRLty6atGtpVTTdtSfnfjmf/WVDgxwGy/8QoJ7cAsv+s8u4BafzT4Z8ED+qnyqOv+Z59tdk/PRb9", + "x0qfR0B6gGNrS59HrueCVzcqSq/dO93UJNfin0mCd/GON5Df/bJ/1fo7qAyVxdrmgrYEz9LMrHxAm/NV", + "lkFnmr9lwxZHcBG3+vFcwbcI6fxw5OHptDWg889ZHP6TxIy64n1ck9MngarrXxjGYPXM1S6WPXvrDKiK", + "FnzJ2o3u9RPslihTbJDJDJwrMS6YWw9/lxmqhvO3xDXvMFfdv6D6I4Dls5jEXLHIJCusxGk5AvbxjSZK", + "Wk0Anku1ao8SwSPyg5LpYzebLfehO1POGFbGGaarQUwNHSw9t9lgQnuP6E4fT2kZHuGCPPue7LA3RmGN", + "CTKzmg/hs2JJsdy+BprcrQ54f9Ri2eRv2WQ+7TLKDdVCnrtqLCTKtZGp3/vTJ2QHqo/NmbB7YUX9GUiy", + "mZJLHrO4NsbeUia4qvstC3pTu6sVKorScV65wMF9Ehmmy4U0f8uzOlsoQmKmXFAY3Na6HPUzhUn8tj/K", + "hQ/AcXvkR/H1CnOa345XdiwlQiVMt4hGSoR43v16zX3J11w1GcrfabXbzofnbDZed8uP6pi29DEKPxS5", + "c3drtn79+aT0cP1FZvM40/myUEjbzOafFwmO7u5+uGtz+esvOAX0GfPKd8VUDg3YFkME8zPEdMdsyRKZ", + "pVCRHN7t9Xu5SnrHvYUx2fHeHsR+L6Q2x0ePHhz23v327v8PAAD//1VBbriI6wEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 3a71084b..809c0df2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1733,8 +1733,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object