diff --git a/Makefile b/Makefile index 4925096..7383f34 100644 --- a/Makefile +++ b/Makefile @@ -184,12 +184,15 @@ dev: ensure-ch-binaries ensure-caddy-binaries lib/system/guest_agent/guest-agent # Run tests (as root for network capabilities, enables caching and parallelism) # Usage: make test - runs all tests # make test TEST=TestCreateInstanceWithNetwork - runs specific test +# make test VERBOSE=1 - runs with verbose output test: ensure-ch-binaries ensure-caddy-binaries lib/system/guest_agent/guest-agent - @if [ -n "$(TEST)" ]; then \ + @VERBOSE_FLAG=""; \ + if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ + if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ - sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) -v -timeout=180s ./...; \ + sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=180s ./...; \ else \ - sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -v -timeout=180s ./...; \ + sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=180s ./...; \ fi # Generate JWT token for testing diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index f511cbf..ec184ab 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -8,6 +8,7 @@ import ( "github.com/onkernel/hypeman/lib/instances" "github.com/onkernel/hypeman/lib/network" "github.com/onkernel/hypeman/lib/oapi" + "github.com/onkernel/hypeman/lib/resources" "github.com/onkernel/hypeman/lib/volumes" ) @@ -20,6 +21,7 @@ type ApiService struct { NetworkManager network.Manager DeviceManager devices.Manager IngressManager ingress.Manager + ResourceManager *resources.Manager } var _ oapi.StrictServerInterface = (*ApiService)(nil) @@ -33,6 +35,7 @@ func New( networkManager network.Manager, deviceManager devices.Manager, ingressManager ingress.Manager, + resourceManager *resources.Manager, ) *ApiService { return &ApiService{ Config: config, @@ -42,5 +45,6 @@ func New( NetworkManager: networkManager, DeviceManager: deviceManager, IngressManager: ingressManager, + ResourceManager: resourceManager, } } diff --git a/cmd/api/api/cp_test.go b/cmd/api/api/cp_test.go index 6b737df..98acf5e 100644 --- a/cmd/api/api/cp_test.go +++ b/cmd/api/api/cp_test.go @@ -46,7 +46,9 @@ func TestCpToAndFromInstance(t *testing.T) { Name: "cp-test", Image: "docker.io/library/nginx:alpine", Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -182,7 +184,9 @@ func TestCpDirectoryToInstance(t *testing.T) { Name: "cp-dir-test", Image: "docker.io/library/nginx:alpine", Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/exec_test.go b/cmd/api/api/exec_test.go index d4a88a6..1042fd8 100644 --- a/cmd/api/api/exec_test.go +++ b/cmd/api/api/exec_test.go @@ -47,7 +47,9 @@ func TestExecInstanceNonTTY(t *testing.T) { Name: "exec-test", Image: "docker.io/library/nginx:alpine", Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -200,7 +202,9 @@ func TestExecWithDebianMinimal(t *testing.T) { Name: "debian-exec-test", Image: "docker.io/library/debian:12-slim", Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 69968b0..5d32916 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "strings" "github.com/c2h5oh/datasize" "github.com/onkernel/hypeman/lib/guest" @@ -15,6 +16,7 @@ import ( mw "github.com/onkernel/hypeman/lib/middleware" "github.com/onkernel/hypeman/lib/network" "github.com/onkernel/hypeman/lib/oapi" + "github.com/onkernel/hypeman/lib/resources" "github.com/samber/lo" ) @@ -82,6 +84,23 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst overlaySize = int64(overlayBytes) } + // Parse disk_io_bps (0 = auto/unlimited) + diskIOBps := int64(0) + if request.Body.DiskIoBps != nil && *request.Body.DiskIoBps != "" { + var ioBpsBytes datasize.ByteSize + // Remove "/s" suffix if present + ioStr := *request.Body.DiskIoBps + ioStr = strings.TrimSuffix(ioStr, "/s") + ioStr = strings.TrimSuffix(ioStr, "ps") + if err := ioBpsBytes.UnmarshalText([]byte(ioStr)); err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_disk_io_bps", + Message: fmt.Sprintf("invalid disk_io_bps format: %v", err), + }, nil + } + diskIOBps = int64(ioBpsBytes) + } + vcpus := 2 if request.Body.Vcpus != nil { vcpus = *request.Body.Vcpus @@ -98,6 +117,33 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst networkEnabled = *request.Body.Network.Enabled } + // Parse network bandwidth limits (0 = auto) + // Supports both bit-based (e.g., "1Gbps") and byte-based (e.g., "125MB/s") formats + var networkBandwidthDownload int64 + var networkBandwidthUpload int64 + if request.Body.Network != nil { + if request.Body.Network.BandwidthDownload != nil && *request.Body.Network.BandwidthDownload != "" { + bw, err := resources.ParseBandwidth(*request.Body.Network.BandwidthDownload) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_bandwidth_download", + Message: fmt.Sprintf("invalid bandwidth_download format: %v", err), + }, nil + } + networkBandwidthDownload = bw + } + if request.Body.Network.BandwidthUpload != nil && *request.Body.Network.BandwidthUpload != "" { + bw, err := resources.ParseBandwidth(*request.Body.Network.BandwidthUpload) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_bandwidth_upload", + Message: fmt.Sprintf("invalid bandwidth_upload format: %v", err), + }, nil + } + networkBandwidthUpload = bw + } + } + // Parse devices (GPU passthrough) var deviceRefs []string if request.Body.Devices != nil { @@ -144,18 +190,36 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst hvType = hypervisor.Type(*request.Body.Hypervisor) } + // Calculate default resource limits when not specified (0 = auto) + // Uses proportional allocation based on CPU: (vcpus / cpuCapacity) * resourceCapacity + if diskIOBps == 0 { + diskIOBps, _ = s.ResourceManager.DefaultDiskIOBandwidth(vcpus) + } + if networkBandwidthDownload == 0 || networkBandwidthUpload == 0 { + defaultDown, defaultUp := s.ResourceManager.DefaultNetworkBandwidth(vcpus) + if networkBandwidthDownload == 0 { + networkBandwidthDownload = defaultDown + } + if networkBandwidthUpload == 0 { + networkBandwidthUpload = defaultUp + } + } + domainReq := instances.CreateInstanceRequest{ - Name: request.Body.Name, - Image: request.Body.Image, - Size: size, - HotplugSize: hotplugSize, - OverlaySize: overlaySize, - Vcpus: vcpus, - Env: env, - NetworkEnabled: networkEnabled, - Devices: deviceRefs, - Volumes: volumes, - Hypervisor: hvType, + Name: request.Body.Name, + Image: request.Body.Image, + Size: size, + HotplugSize: hotplugSize, + OverlaySize: overlaySize, + Vcpus: vcpus, + DiskIOBps: diskIOBps, + NetworkBandwidthDownload: networkBandwidthDownload, + NetworkBandwidthUpload: networkBandwidthUpload, + Env: env, + NetworkEnabled: networkEnabled, + Devices: deviceRefs, + Volumes: volumes, + Hypervisor: hvType, } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) @@ -539,14 +603,29 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { hotplugSizeStr := datasize.ByteSize(inst.HotplugSize).HR() overlaySizeStr := datasize.ByteSize(inst.OverlaySize).HR() - // Build network object with ip/mac nested inside + // Format bandwidth as human-readable (bytes/s to rate string) + var downloadBwStr, uploadBwStr *string + if inst.NetworkBandwidthDownload > 0 { + s := datasize.ByteSize(inst.NetworkBandwidthDownload).HR() + "/s" + downloadBwStr = &s + } + if inst.NetworkBandwidthUpload > 0 { + s := datasize.ByteSize(inst.NetworkBandwidthUpload).HR() + "/s" + uploadBwStr = &s + } + + // Build network object with ip/mac and bandwidth nested inside netObj := &struct { - Enabled *bool `json:"enabled,omitempty"` - Ip *string `json:"ip"` - Mac *string `json:"mac"` - Name *string `json:"name,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Ip *string `json:"ip"` + Mac *string `json:"mac"` + Name *string `json:"name,omitempty"` }{ - Enabled: lo.ToPtr(inst.NetworkEnabled), + Enabled: lo.ToPtr(inst.NetworkEnabled), + BandwidthDownload: downloadBwStr, + BandwidthUpload: uploadBwStr, } if inst.NetworkEnabled { netObj.Name = lo.ToPtr("default") @@ -557,6 +636,13 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { // Convert hypervisor type hvType := oapi.InstanceHypervisor(inst.HypervisorType) + // Format disk I/O as human-readable + var diskIoBpsStr *string + if inst.DiskIOBps > 0 { + s := datasize.ByteSize(inst.DiskIOBps).HR() + "/s" + diskIoBpsStr = &s + } + oapiInst := oapi.Instance{ Id: inst.Id, Name: inst.Name, @@ -567,6 +653,7 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { HotplugSize: lo.ToPtr(hotplugSizeStr), OverlaySize: lo.ToPtr(overlaySizeStr), Vcpus: lo.ToPtr(inst.Vcpus), + DiskIoBps: diskIoBpsStr, Network: netObj, CreatedAt: inst.CreatedAt, StartedAt: inst.StartedAt, diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 82f3886..ffe45a0 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -65,7 +65,9 @@ func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { HotplugSize: &hotplugSize, OverlaySize: &overlaySize, Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -109,7 +111,9 @@ func TestCreateInstance_InvalidSizeFormat(t *testing.T) { Image: "docker.io/library/alpine:latest", Size: &invalidSize, Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -150,7 +154,9 @@ func TestInstanceLifecycle_StopStart(t *testing.T) { Name: "test-lifecycle", Image: "docker.io/library/nginx:alpine", Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -208,11 +214,11 @@ func waitForState(t *testing.T, svc *ApiService, instanceID string, expectedStat inst, err := svc.InstanceManager.GetInstance(ctx(), instanceID) require.NoError(t, err) - if string(inst.State) == expectedState { - t.Logf("Instance reached %s state", expectedState) - return - } - t.Logf("Instance state: %s (waiting for %s)", inst.State, expectedState) + if string(inst.State) == expectedState { + t.Logf("Instance reached %s state", expectedState) + return + } + t.Logf("Instance state: %s (waiting for %s)", inst.State, expectedState) time.Sleep(100 * time.Millisecond) } t.Fatalf("Timeout waiting for instance to reach %s state", expectedState) diff --git a/cmd/api/api/registry_test.go b/cmd/api/api/registry_test.go index 1e9e255..45e6264 100644 --- a/cmd/api/api/registry_test.go +++ b/cmd/api/api/registry_test.go @@ -135,7 +135,9 @@ func TestRegistryPushAndCreateInstance(t *testing.T) { Name: "test-pushed-image", Image: imageName, Network: &struct { - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go new file mode 100644 index 0000000..f79faec --- /dev/null +++ b/cmd/api/api/resources.go @@ -0,0 +1,72 @@ +package api + +import ( + "context" + + "github.com/onkernel/hypeman/lib/oapi" + "github.com/onkernel/hypeman/lib/resources" +) + +// GetResources returns host resource capacity and allocations +func (s *ApiService) GetResources(ctx context.Context, _ oapi.GetResourcesRequestObject) (oapi.GetResourcesResponseObject, error) { + if s.ResourceManager == nil { + return oapi.GetResources500JSONResponse{ + Code: "internal_error", + Message: "Resource manager not initialized", + }, nil + } + + status, err := s.ResourceManager.GetFullStatus(ctx) + if err != nil { + return oapi.GetResources500JSONResponse{ + Code: "internal_error", + Message: err.Error(), + }, nil + } + + // Convert to API response + resp := oapi.Resources{ + Cpu: convertResourceStatus(status.CPU), + Memory: convertResourceStatus(status.Memory), + Disk: convertResourceStatus(status.Disk), + Network: convertResourceStatus(status.Network), + Allocations: make([]oapi.ResourceAllocation, 0, len(status.Allocations)), + } + + // Add disk breakdown if available + if status.DiskDetail != nil { + resp.DiskBreakdown = &oapi.DiskBreakdown{ + ImagesBytes: &status.DiskDetail.Images, + OciCacheBytes: &status.DiskDetail.OCICache, + VolumesBytes: &status.DiskDetail.Volumes, + OverlaysBytes: &status.DiskDetail.Overlays, + } + } + + // Add per-instance allocations + for _, alloc := range status.Allocations { + resp.Allocations = append(resp.Allocations, oapi.ResourceAllocation{ + InstanceId: &alloc.InstanceID, + InstanceName: &alloc.InstanceName, + Cpu: &alloc.CPU, + MemoryBytes: &alloc.MemoryBytes, + DiskBytes: &alloc.DiskBytes, + NetworkDownloadBps: &alloc.NetworkDownloadBps, + NetworkUploadBps: &alloc.NetworkUploadBps, + }) + } + + return oapi.GetResources200JSONResponse(resp), nil +} + +func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { + return oapi.ResourceStatus{ + Type: string(rs.Type), + Capacity: rs.Capacity, + EffectiveLimit: rs.EffectiveLimit, + Allocated: rs.Allocated, + Available: rs.Available, + OversubRatio: rs.OversubRatio, + Source: &rs.Source, + } +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 2d43b52..438c601 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "runtime/debug" "strconv" @@ -104,6 +105,23 @@ type Config struct { // Hypervisor configuration DefaultHypervisor string // Default hypervisor type: "cloud-hypervisor" or "qemu" + + // Oversubscription ratios (1.0 = no oversubscription, 2.0 = 2x oversubscription) + OversubCPU float64 // CPU oversubscription ratio + OversubMemory float64 // Memory oversubscription ratio + OversubDisk float64 // Disk oversubscription ratio + OversubNetwork float64 // Network oversubscription ratio + OversubDiskIO float64 // Disk I/O oversubscription ratio + + // Network rate limiting + UploadBurstMultiplier int // Multiplier for upload burst ceiling vs guaranteed rate (default: 4) + DownloadBurstMultiplier int // Multiplier for download burst bucket vs rate (default: 4) + + // Resource capacity limits (empty = auto-detect from host) + DiskLimit string // Hard disk limit for DataDir, e.g. "500GB" + NetworkLimit string // Hard network limit, e.g. "10Gbps" (empty = detect from uplink speed) + DiskIOLimit string // Hard disk I/O limit, e.g. "500MB/s" (empty = auto-detect from disk type) + MaxImageStorage float64 // Max image storage as fraction of disk (0.2 = 20%), counts OCI cache + rootfs } // Load loads configuration from environment variables @@ -169,6 +187,23 @@ func Load() *Config { // Hypervisor configuration DefaultHypervisor: getEnv("DEFAULT_HYPERVISOR", "cloud-hypervisor"), + + // Oversubscription ratios (1.0 = no oversubscription) + OversubCPU: getEnvFloat("OVERSUB_CPU", 4.0), + OversubMemory: getEnvFloat("OVERSUB_MEMORY", 1.0), + OversubDisk: getEnvFloat("OVERSUB_DISK", 1.0), + OversubNetwork: getEnvFloat("OVERSUB_NETWORK", 2.0), + OversubDiskIO: getEnvFloat("OVERSUB_DISK_IO", 2.0), + + // Network rate limiting + UploadBurstMultiplier: getEnvInt("UPLOAD_BURST_MULTIPLIER", 4), + DownloadBurstMultiplier: getEnvInt("DOWNLOAD_BURST_MULTIPLIER", 4), + + // Resource capacity limits (empty = auto-detect) + DiskLimit: getEnv("DISK_LIMIT", ""), + NetworkLimit: getEnv("NETWORK_LIMIT", ""), + DiskIOLimit: getEnv("DISK_IO_LIMIT", ""), + MaxImageStorage: getEnvFloat("MAX_IMAGE_STORAGE", 0.2), // 20% of disk by default } return cfg @@ -198,3 +233,40 @@ func getEnvBool(key string, defaultValue bool) bool { } return defaultValue } + +func getEnvFloat(key string, defaultValue float64) float64 { + if value := os.Getenv(key); value != "" { + if floatVal, err := strconv.ParseFloat(value, 64); err == nil { + return floatVal + } + } + return defaultValue +} + +// Validate checks configuration values for correctness. +// Returns an error if any configuration value is invalid. +func (c *Config) Validate() error { + // Validate oversubscription ratios are positive + if c.OversubCPU <= 0 { + return fmt.Errorf("OVERSUB_CPU must be positive, got %v", c.OversubCPU) + } + if c.OversubMemory <= 0 { + return fmt.Errorf("OVERSUB_MEMORY must be positive, got %v", c.OversubMemory) + } + if c.OversubDisk <= 0 { + return fmt.Errorf("OVERSUB_DISK must be positive, got %v", c.OversubDisk) + } + if c.OversubNetwork <= 0 { + return fmt.Errorf("OVERSUB_NETWORK must be positive, got %v", c.OversubNetwork) + } + if c.OversubDiskIO <= 0 { + return fmt.Errorf("OVERSUB_DISK_IO must be positive, got %v", c.OversubDiskIO) + } + if c.UploadBurstMultiplier < 1 { + return fmt.Errorf("UPLOAD_BURST_MULTIPLIER must be >= 1, got %v", c.UploadBurstMultiplier) + } + if c.DownloadBurstMultiplier < 1 { + return fmt.Errorf("DOWNLOAD_BURST_MULTIPLIER must be >= 1, got %v", c.DownloadBurstMultiplier) + } + return nil +} diff --git a/cmd/api/main.go b/cmd/api/main.go index dfb556b..18e66a6 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -45,6 +45,11 @@ func run() error { // Load config early for OTel initialization cfg := config.Load() + // Validate configuration before proceeding + if err := cfg.Validate(); err != nil { + return fmt.Errorf("invalid configuration: %w", err) + } + // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ Enabled: cfg.OtelEnabled, @@ -176,7 +181,12 @@ func run() error { logger.Error("failed to initialize network manager", "error", err) return fmt.Errorf("initialize network manager: %w", err) } - logger.Info("Network manager initialized") + + // Set up HTB qdisc on bridge for network fair sharing + networkCapacity := app.ResourceManager.NetworkCapacity() + if err := app.NetworkManager.SetupHTB(app.Ctx, networkCapacity); err != nil { + logger.Warn("failed to setup HTB on bridge (network rate limiting disabled)", "error", err) + } // Reconcile device state (clears orphaned attachments from crashed VMs) // Set up liveness checker so device reconciliation can accurately detect orphaned attachments diff --git a/cmd/api/wire.go b/cmd/api/wire.go index dfa2fc1..d9e734a 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -16,6 +16,7 @@ import ( "github.com/onkernel/hypeman/lib/network" "github.com/onkernel/hypeman/lib/providers" "github.com/onkernel/hypeman/lib/registry" + "github.com/onkernel/hypeman/lib/resources" "github.com/onkernel/hypeman/lib/system" "github.com/onkernel/hypeman/lib/volumes" ) @@ -32,6 +33,7 @@ type application struct { InstanceManager instances.Manager VolumeManager volumes.Manager IngressManager ingress.Manager + ResourceManager *resources.Manager Registry *registry.Registry ApiService *api.ApiService } @@ -50,6 +52,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideInstanceManager, providers.ProvideVolumeManager, providers.ProvideIngressManager, + providers.ProvideResourceManager, providers.ProvideRegistry, api.New, wire.Struct(new(application), "*"), diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 6b3e81a..0ba6a77 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -8,8 +8,6 @@ package main import ( "context" - "log/slog" - "github.com/onkernel/hypeman/cmd/api/api" "github.com/onkernel/hypeman/cmd/api/config" "github.com/onkernel/hypeman/lib/devices" @@ -19,9 +17,13 @@ import ( "github.com/onkernel/hypeman/lib/network" "github.com/onkernel/hypeman/lib/providers" "github.com/onkernel/hypeman/lib/registry" + "github.com/onkernel/hypeman/lib/resources" "github.com/onkernel/hypeman/lib/system" "github.com/onkernel/hypeman/lib/volumes" + "log/slog" +) +import ( _ "embed" ) @@ -52,11 +54,15 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + resourcesManager, err := providers.ProvideResourceManager(context, config, paths, manager, instancesManager, volumesManager) + if err != nil { + return nil, nil, err + } registry, err := providers.ProvideRegistry(paths, manager) if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager) + apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, resourcesManager) mainApplication := &application{ Ctx: context, Logger: logger, @@ -68,6 +74,7 @@ func initializeApp() (*application, func(), error) { InstanceManager: instancesManager, VolumeManager: volumesManager, IngressManager: ingressManager, + ResourceManager: resourcesManager, Registry: registry, ApiService: apiService, } @@ -89,6 +96,7 @@ type application struct { InstanceManager instances.Manager VolumeManager volumes.Manager IngressManager ingress.Manager + ResourceManager *resources.Manager Registry *registry.Registry ApiService *api.ApiService } diff --git a/go.sum b/go.sum index 3edd372..12ceb44 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU= github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y= +github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go index 4410ff4..fc80ac2 100644 --- a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go +++ b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go @@ -38,6 +38,7 @@ func (c *CloudHypervisor) Capabilities() hypervisor.Capabilities { SupportsPause: true, SupportsVsock: true, SupportsGPUPassthrough: true, + SupportsDiskIOLimit: true, } } diff --git a/lib/hypervisor/cloudhypervisor/config.go b/lib/hypervisor/cloudhypervisor/config.go index 22d4a9a..d38308b 100644 --- a/lib/hypervisor/cloudhypervisor/config.go +++ b/lib/hypervisor/cloudhypervisor/config.go @@ -48,6 +48,21 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { if d.Readonly { disk.Readonly = ptr(true) } + if d.IOBps > 0 { + // Token bucket: Size is refilled every RefillTime ms + // Rate = Size / RefillTime * 1000 = Size bytes/sec (when RefillTime = 1000) + burstBps := d.IOBurstBps + if burstBps <= 0 { + burstBps = d.IOBps + } + disk.RateLimiterConfig = &vmm.RateLimiterConfig{ + Bandwidth: &vmm.TokenBucket{ + Size: d.IOBps, // sustained rate (bytes/sec with 1s refill) + RefillTime: 1000, // refill over 1 second + OneTimeBurst: ptr(burstBps - d.IOBps), // extra burst capacity + }, + } + } disks = append(disks, disk) } diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index 04f8040..2682de4 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -41,8 +41,10 @@ type CPUTopology struct { // DiskConfig represents a disk attached to the VM type DiskConfig struct { - Path string - Readonly bool + Path string + Readonly bool + IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited) + IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps) } // NetworkConfig represents a network interface attached to the VM diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index a92f832..4306f08 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -124,6 +124,9 @@ type Capabilities struct { // SupportsGPUPassthrough indicates if PCI device passthrough is available SupportsGPUPassthrough bool + + // SupportsDiskIOLimit indicates if disk I/O rate limiting is available + SupportsDiskIOLimit bool } // VsockDialer provides vsock connectivity to a guest VM. diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 5f3b457..57c539a 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -40,6 +40,12 @@ func BuildArgs(cfg hypervisor.VMConfig) []string { if disk.Readonly { driveOpts += ",readonly=on" } + if disk.IOBps > 0 { + driveOpts += fmt.Sprintf(",throttling.bps-total=%d", disk.IOBps) + if disk.IOBurstBps > 0 && disk.IOBurstBps > disk.IOBps { + driveOpts += fmt.Sprintf(",throttling.bps-total-max=%d", disk.IOBurstBps) + } + } args = append(args, "-drive", driveOpts) args = append(args, "-device", fmt.Sprintf("virtio-blk-pci,drive=drive%d", i)) } diff --git a/lib/hypervisor/qemu/qemu.go b/lib/hypervisor/qemu/qemu.go index a77dd71..e54f2fd 100644 --- a/lib/hypervisor/qemu/qemu.go +++ b/lib/hypervisor/qemu/qemu.go @@ -43,6 +43,7 @@ func (q *QEMU) Capabilities() hypervisor.Capabilities { SupportsPause: true, SupportsVsock: true, SupportsGPUPassthrough: true, + SupportsDiskIOLimit: true, } } diff --git a/lib/images/manager.go b/lib/images/manager.go index 639de5e..42cfd83 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/onkernel/hypeman/lib/paths" "go.opentelemetry.io/otel/metric" ) @@ -31,6 +32,12 @@ type Manager interface { GetImage(ctx context.Context, name string) (*Image, error) DeleteImage(ctx context.Context, name string) error RecoverInterruptedBuilds() + // TotalImageBytes returns the total size of all ready images on disk. + // Used by the resource manager for disk capacity tracking. + TotalImageBytes(ctx context.Context) (int64, error) + // TotalOCICacheBytes returns the total size of the OCI layer cache. + // Used by the resource manager for disk capacity tracking. + TotalOCICacheBytes(ctx context.Context) (int64, error) } type manager struct { @@ -382,3 +389,76 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { return deleteTag(m.paths, repository, tag) } + +// TotalImageBytes returns the total size of all ready images on disk. +func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { + images, err := m.ListImages(ctx) + if err != nil { + return 0, err + } + + var total int64 + for _, img := range images { + if img.Status == StatusReady && img.SizeBytes != nil { + total += *img.SizeBytes + } + } + return total, nil +} + +// TotalOCICacheBytes returns the total size of the OCI layer cache. +// Uses OCI layout metadata instead of walking the filesystem for efficiency. +func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { + path, err := layout.FromPath(m.paths.SystemOCICache()) + if err != nil { + return 0, nil // No cache yet + } + + index, err := path.ImageIndex() + if err != nil { + return 0, nil // Empty or invalid cache + } + + manifest, err := index.IndexManifest() + if err != nil { + return 0, nil + } + + // Collect unique blob digests and sizes (layers are shared/deduplicated) + blobSizes := make(map[string]int64) + + for _, desc := range manifest.Manifests { + // Count the manifest blob itself + blobSizes[desc.Digest.String()] = desc.Size + + // Get image to access layers and config + img, err := path.Image(desc.Digest) + if err != nil { + continue + } + + // Count config blob + if configDigest, err := img.ConfigName(); err == nil { + if configFile, err := img.RawConfigFile(); err == nil { + blobSizes[configDigest.String()] = int64(len(configFile)) + } + } + + // Count layer blobs + if layers, err := img.Layers(); err == nil { + for _, layer := range layers { + if digest, err := layer.Digest(); err == nil { + if size, err := layer.Size(); err == nil { + blobSizes[digest.String()] = size + } + } + } + } + } + + var total int64 + for _, size := range blobSizes { + total += size + } + return total, nil +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 3938fff..194a3a9 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -287,26 +287,29 @@ func (m *manager) createInstance( // 11. Create instance metadata stored := &StoredMetadata{ - Id: id, - Name: req.Name, - Image: req.Image, - Size: size, - HotplugSize: hotplugSize, - OverlaySize: overlaySize, - Vcpus: vcpus, - Env: req.Env, - NetworkEnabled: req.NetworkEnabled, - CreatedAt: time.Now(), - StartedAt: nil, - StoppedAt: nil, - KernelVersion: string(kernelVer), - HypervisorType: hvType, - HypervisorVersion: hvVersion, - SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), - DataDir: m.paths.InstanceDir(id), - VsockCID: vsockCID, - VsockSocket: vsockSocket, - Devices: resolvedDeviceIDs, + Id: id, + Name: req.Name, + Image: req.Image, + Size: size, + HotplugSize: hotplugSize, + OverlaySize: overlaySize, + Vcpus: vcpus, + NetworkBandwidthDownload: req.NetworkBandwidthDownload, // Will be set by caller if using resource manager + NetworkBandwidthUpload: req.NetworkBandwidthUpload, // Will be set by caller if using resource manager + DiskIOBps: req.DiskIOBps, // Will be set by caller if using resource manager + Env: req.Env, + NetworkEnabled: req.NetworkEnabled, + CreatedAt: time.Now(), + StartedAt: nil, + StoppedAt: nil, + KernelVersion: string(kernelVer), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + VsockCID: vsockCID, + VsockSocket: vsockSocket, + Devices: resolvedDeviceIDs, } // 12. Ensure directories @@ -326,10 +329,14 @@ func (m *manager) createInstance( // 14. Allocate network (if network enabled) var netConfig *network.NetworkConfig if networkName != "" { - log.DebugContext(ctx, "allocating network", "instance_id", id, "network", networkName) + log.DebugContext(ctx, "allocating network", "instance_id", id, "network", networkName, + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) netConfig, err = m.networkManager.CreateAllocation(ctx, network.AllocateRequest{ - InstanceID: id, - InstanceName: req.Name, + InstanceID: id, + InstanceName: req.Name, + DownloadBps: stored.NetworkBandwidthDownload, + UploadBps: stored.NetworkBandwidthUpload, + UploadCeilBps: stored.NetworkBandwidthUpload * int64(m.networkManager.GetUploadBurstMultiplier()), }) if err != nil { log.ErrorContext(ctx, "failed to allocate network", "instance_id", id, "network", networkName, "error", err) @@ -598,13 +605,20 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima return hypervisor.VMConfig{}, err } + // Get disk I/O limits (same for all disks in this VM) + ioBps := inst.DiskIOBps + burstBps := ioBps * 4 // Burst is 4x sustained + if ioBps <= 0 { + burstBps = 0 + } + disks := []hypervisor.DiskConfig{ // Rootfs (from image, read-only) - {Path: rootfsPath, Readonly: true}, + {Path: rootfsPath, Readonly: true, IOBps: ioBps, IOBurstBps: burstBps}, // Overlay disk (writable) - {Path: m.paths.InstanceOverlay(inst.Id), Readonly: false}, + {Path: m.paths.InstanceOverlay(inst.Id), Readonly: false, IOBps: ioBps, IOBurstBps: burstBps}, // Config disk (read-only) - {Path: m.paths.InstanceConfigDisk(inst.Id), Readonly: true}, + {Path: m.paths.InstanceConfigDisk(inst.Id), Readonly: true, IOBps: ioBps, IOBurstBps: burstBps}, } // Add attached volumes as additional disks @@ -613,19 +627,25 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima if volAttach.Overlay { // Base volume is always read-only when overlay is enabled disks = append(disks, hypervisor.DiskConfig{ - Path: volumePath, - Readonly: true, + Path: volumePath, + Readonly: true, + IOBps: ioBps, + IOBurstBps: burstBps, }) // Overlay disk is writable overlayPath := m.paths.InstanceVolumeOverlay(inst.Id, volAttach.VolumeID) disks = append(disks, hypervisor.DiskConfig{ - Path: overlayPath, - Readonly: false, + Path: overlayPath, + Readonly: false, + IOBps: ioBps, + IOBurstBps: burstBps, }) } else { disks = append(disks, hypervisor.DiskConfig{ - Path: volumePath, - Readonly: volAttach.Readonly, + Path: volumePath, + Readonly: volAttach.Readonly, + IOBps: ioBps, + IOBurstBps: burstBps, }) } } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 915879d..ebaf9a7 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -12,6 +12,7 @@ import ( "github.com/onkernel/hypeman/lib/images" "github.com/onkernel/hypeman/lib/network" "github.com/onkernel/hypeman/lib/paths" + "github.com/onkernel/hypeman/lib/resources" "github.com/onkernel/hypeman/lib/system" "github.com/onkernel/hypeman/lib/volumes" "go.opentelemetry.io/otel/metric" @@ -34,6 +35,9 @@ type Manager interface { RotateLogs(ctx context.Context, maxBytes int64, maxFiles int) error AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error) DetachVolume(ctx context.Context, id string, volumeId string) (*Instance, error) + // ListInstanceAllocations returns resource allocations for all instances. + // Used by the resource manager for capacity tracking. + ListInstanceAllocations(ctx context.Context) ([]resources.InstanceAllocation, error) } // ResourceLimits contains configurable resource limits for instances @@ -281,3 +285,46 @@ func (m *manager) AttachVolume(ctx context.Context, id string, volumeId string, func (m *manager) DetachVolume(ctx context.Context, id string, volumeId string) (*Instance, error) { return nil, fmt.Errorf("detach volume not yet implemented") } + +// ListInstanceAllocations returns resource allocations for all instances. +// Used by the resource manager for capacity tracking. +func (m *manager) ListInstanceAllocations(ctx context.Context) ([]resources.InstanceAllocation, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return nil, err + } + + allocations := make([]resources.InstanceAllocation, 0, len(instances)) + for _, inst := range instances { + // Calculate volume bytes and volume overlay bytes separately + var volumeBytes int64 + var volumeOverlayBytes int64 + for _, vol := range inst.Volumes { + // Get actual volume size from volume manager + if m.volumeManager != nil { + if volume, err := m.volumeManager.GetVolume(ctx, vol.VolumeID); err == nil { + volumeBytes += int64(volume.SizeGb) * 1024 * 1024 * 1024 + } + } + // Track overlay size separately for overlay volumes + if vol.Overlay { + volumeOverlayBytes += vol.OverlaySize + } + } + + allocations = append(allocations, resources.InstanceAllocation{ + ID: inst.Id, + Name: inst.Name, + Vcpus: inst.Vcpus, + MemoryBytes: inst.Size + inst.HotplugSize, + OverlayBytes: inst.OverlaySize, + VolumeOverlayBytes: volumeOverlayBytes, + NetworkDownloadBps: inst.NetworkBandwidthDownload, + NetworkUploadBps: inst.NetworkBandwidthUpload, + State: string(inst.State), + VolumeBytes: volumeBytes, + }) + } + + return allocations, nil +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 69590b5..6a5ac61 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -60,8 +60,9 @@ func (m *manager) restoreInstance( if m.metrics != nil && m.metrics.tracer != nil { ctx, networkSpan = m.metrics.tracer.Start(ctx, "RestoreNetwork") } - log.DebugContext(ctx, "recreating network for restore", "instance_id", id, "network", "default") - if err := m.networkManager.RecreateAllocation(ctx, id); err != nil { + log.InfoContext(ctx, "recreating network for restore", "instance_id", id, "network", "default", + "download_bps", stored.NetworkBandwidthDownload, "upload_bps", stored.NetworkBandwidthUpload) + if err := m.networkManager.RecreateAllocation(ctx, id, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload); err != nil { if networkSpan != nil { networkSpan.End() } @@ -78,7 +79,7 @@ func (m *manager) restoreInstance( if m.metrics != nil && m.metrics.tracer != nil { ctx, restoreSpan = m.metrics.tracer.Start(ctx, "RestoreFromSnapshot") } - log.DebugContext(ctx, "restoring from snapshot", "instance_id", id, "snapshot_dir", snapshotDir, "hypervisor", stored.HypervisorType) + log.InfoContext(ctx, "restoring from snapshot", "instance_id", id, "snapshot_dir", snapshotDir, "hypervisor", stored.HypervisorType) pid, hv, err := m.restoreFromSnapshot(ctx, stored, snapshotDir) if restoreSpan != nil { restoreSpan.End() @@ -101,7 +102,7 @@ func (m *manager) restoreInstance( if m.metrics != nil && m.metrics.tracer != nil { ctx, resumeSpan = m.metrics.tracer.Start(ctx, "ResumeVM") } - log.DebugContext(ctx, "resuming VM", "instance_id", id) + log.InfoContext(ctx, "resuming VM", "instance_id", id) if err := hv.Resume(ctx); err != nil { if resumeSpan != nil { resumeSpan.End() @@ -120,7 +121,7 @@ func (m *manager) restoreInstance( } // 8. Delete snapshot after successful restore - log.DebugContext(ctx, "deleting snapshot after successful restore", "instance_id", id) + log.InfoContext(ctx, "deleting snapshot after successful restore", "instance_id", id) os.RemoveAll(snapshotDir) // Best effort, ignore errors // 9. Update timestamp diff --git a/lib/instances/types.go b/lib/instances/types.go index 5d7d05f..0d0c954 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -36,10 +36,13 @@ type StoredMetadata struct { Image string // OCI reference // Resources (matching Cloud Hypervisor terminology) - Size int64 // Base memory in bytes - HotplugSize int64 // Hotplug memory in bytes - OverlaySize int64 // Overlay disk size in bytes - Vcpus int + Size int64 // Base memory in bytes + HotplugSize int64 // Hotplug memory in bytes + OverlaySize int64 // Overlay disk size in bytes + Vcpus int + NetworkBandwidthDownload int64 // Download rate limit in bytes/sec (external→VM), 0 = auto + NetworkBandwidthUpload int64 // Upload rate limit in bytes/sec (VM→external), 0 = auto + DiskIOBps int64 // Disk I/O rate limit in bytes/sec, 0 = auto // Configuration Env map[string]string @@ -93,17 +96,20 @@ func (i *Instance) GetHypervisorType() string { // CreateInstanceRequest is the domain request for creating an instance type CreateInstanceRequest struct { - Name string // Required - Image string // Required: OCI reference - Size int64 // Base memory in bytes (default: 1GB) - HotplugSize int64 // Hotplug memory in bytes (default: 3GB) - OverlaySize int64 // Overlay disk size in bytes (default: 10GB) - Vcpus int // Default 2 - Env map[string]string // Optional environment variables - NetworkEnabled bool // Whether to enable networking (uses default network) - Devices []string // Device IDs or names to attach (GPU passthrough) - Volumes []VolumeAttachment // Volumes to attach at creation time - Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) + Name string // Required + Image string // Required: OCI reference + Size int64 // Base memory in bytes (default: 1GB) + HotplugSize int64 // Hotplug memory in bytes (default: 3GB) + OverlaySize int64 // Overlay disk size in bytes (default: 10GB) + Vcpus int // Default 2 + NetworkBandwidthDownload int64 // Download rate limit bytes/sec (0 = auto, proportional to CPU) + NetworkBandwidthUpload int64 // Upload rate limit bytes/sec (0 = auto, proportional to CPU) + DiskIOBps int64 // Disk I/O rate limit bytes/sec (0 = auto, proportional to CPU) + Env map[string]string // Optional environment variables + NetworkEnabled bool // Whether to enable networking (uses default network) + Devices []string // Device IDs or names to attach (GPU passthrough) + Volumes []VolumeAttachment // Volumes to attach at creation time + Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/network/README.md b/lib/network/README.md index c1bdd89..1e77153 100644 --- a/lib/network/README.md +++ b/lib/network/README.md @@ -185,10 +185,74 @@ sudo setcap 'cap_net_admin,cap_net_bind_service=+eip' /path/to/hypeman 1. Derive allocation from snapshot config.json 2. Recreate TAP device with same name 3. Attach to bridge with isolation mode +4. Reapply rate limits from instance metadata ### ReleaseAllocation (for shutdown/delete) 1. Derive current allocation -2. Delete TAP device +2. Remove HTB class from bridge (if upload limiting enabled) +3. Delete TAP device + +## Bidirectional Rate Limiting + +Network bandwidth is limited separately for download and upload directions: + +``` + Internet + │ + ┌────┴────┐ + │ eth0 │ + │ (uplink)│ + └────┬────┘ + │ + ┌───────────────────┴───────────────────┐ + │ Bridge (vmbr0) │ + │ HTB qdisc for upload shaping │ + │ ┌────────┬────────┐ │ + │ │ 1:a1b2 │ 1:c3d4 │ │ + │ │ VM-A │ VM-B │ │ + │ │ rate+ │ rate+ │ │ + │ │ ceil │ ceil │ │ + │ └────┬───┴────┬───┘ │ + └───────┼────────┼──────────────────────┘ + │ │ + ┌────┴───┐┌───┴────┐ + │ TAP-A ││ TAP-B │ + │ + TBF ││ + TBF │ (download shaping) + └────┬───┘└───┬────┘ + │ │ + ┌───┴───┐┌───┴───┐ + │ VM-A ││ VM-B │ + └───────┘└───────┘ +``` + +### Download (external → VM) +- **Method:** TBF (Token Bucket Filter) on TAP device egress +- **Behavior:** Queues packets to smooth traffic, doesn't drop +- **Per-VM:** Each TAP gets its own shaper, independent + +### Upload (VM → external) +- **Method:** HTB (Hierarchical Token Bucket) on bridge egress +- **Behavior:** Fair sharing with guaranteed rates and burst ceilings +- **Per-VM:** Each VM gets an HTB class with: + - `rate`: Guaranteed bandwidth (always available) + - `ceil`: Burst ceiling (can use more when others are idle) + - `fq_codel`: Leaf qdisc for low latency + +### Why Different Methods? + +| Direction | Bottleneck | Solution | +|-----------|------------|----------| +| Download | Physical NIC ingress | Shape before delivery to each TAP | +| Upload | Physical NIC egress (shared) | Centralized HTB for fair arbitration | + +**Policing (drop-based) was rejected** because it causes TCP to oscillate due to congestion control reacting to packet loss. Shaping (queue-based) provides smoother, more predictable throughput. + +### Default Limits + +When not specified in the create request: +- Both download and upload = `(vcpus / cpu_capacity) * network_capacity` +- Symmetric by default +- Upload ceiling = 4x guaranteed rate (configurable via `UPLOAD_BURST_MULTIPLIER`) Note: In case of unexpected scenarios like power loss, straggler TAP devices may persist until manual cleanup or host reboot. diff --git a/lib/network/allocate.go b/lib/network/allocate.go index b3f3590..b747ec4 100644 --- a/lib/network/allocate.go +++ b/lib/network/allocate.go @@ -55,8 +55,8 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N // 5. Generate TAP name (tap-{first8chars-of-id}) tap := generateTAPName(req.InstanceID) - // 6. Create TAP device - if err := m.createTAPDevice(tap, network.Bridge, network.Isolated); err != nil { + // 6. Create TAP device with bidirectional rate limiting + if err := m.createTAPDevice(tap, network.Bridge, network.Isolated, req.DownloadBps, req.UploadBps, req.UploadCeilBps); err != nil { return nil, fmt.Errorf("create TAP device: %w", err) } m.recordTAPOperation(ctx, "create") @@ -67,7 +67,9 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N "network", "default", "ip", ip, "mac", mac, - "tap", tap) + "tap", tap, + "download_bps", req.DownloadBps, + "upload_bps", req.UploadBps) // 7. Calculate netmask from subnet _, ipNet, _ := net.ParseCIDR(network.Subnet) @@ -89,7 +91,7 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N // 1. Doesn't allocate new IPs (reuses existing from snapshot) // 2. Is already protected by instance-level locking // 3. Uses deterministic TAP names that can't conflict -func (m *manager) RecreateAllocation(ctx context.Context, instanceID string) error { +func (m *manager) RecreateAllocation(ctx context.Context, instanceID string, downloadBps, uploadBps int64) error { log := logger.FromContext(ctx) // 1. Derive allocation from snapshot @@ -108,8 +110,9 @@ func (m *manager) RecreateAllocation(ctx context.Context, instanceID string) err return fmt.Errorf("get default network: %w", err) } - // 3. Recreate TAP device with same name - if err := m.createTAPDevice(alloc.TAPDevice, network.Bridge, network.Isolated); err != nil { + // 3. Recreate TAP device with same name and rate limits from instance metadata + uploadCeilBps := uploadBps * int64(m.GetUploadBurstMultiplier()) + if err := m.createTAPDevice(alloc.TAPDevice, network.Bridge, network.Isolated, downloadBps, uploadBps, uploadCeilBps); err != nil { return fmt.Errorf("create TAP device: %w", err) } m.recordTAPOperation(ctx, "create") @@ -117,7 +120,9 @@ func (m *manager) RecreateAllocation(ctx context.Context, instanceID string) err log.InfoContext(ctx, "recreated network for restore", "instance_id", instanceID, "network", "default", - "tap", alloc.TAPDevice) + "tap", alloc.TAPDevice, + "download_bps", downloadBps, + "upload_bps", uploadBps) return nil } diff --git a/lib/network/bridge.go b/lib/network/bridge.go index 75c2b45..b6fe80f 100644 --- a/lib/network/bridge.go +++ b/lib/network/bridge.go @@ -3,6 +3,7 @@ package network import ( "context" "fmt" + "hash/fnv" "net" "os" "os/exec" @@ -188,6 +189,12 @@ const ( commentFwdIn = "hypeman-fwd-in" ) +// HTB handles for traffic control +const ( + htbRootHandle = "1:" // Root qdisc handle + htbRootClassID = "1:1" // Root class for total capacity +) + // getUplinkInterface returns the uplink interface for NAT/forwarding. // Uses explicit config if set, otherwise auto-detects from default route. func (m *manager) getUplinkInterface() (string, error) { @@ -416,8 +423,10 @@ func (m *manager) deleteForwardRuleByComment(comment string) { } } -// createTAPDevice creates TAP device and attaches to bridge -func (m *manager) createTAPDevice(tapName, bridgeName string, isolated bool) error { +// createTAPDevice creates TAP device and attaches to bridge. +// downloadBps: rate limit for download (external→VM), applied as TBF on TAP egress +// uploadBps/uploadCeilBps: rate limit for upload (VM→external), applied as HTB class on bridge +func (m *manager) createTAPDevice(tapName, bridgeName string, isolated bool, downloadBps, uploadBps, uploadCeilBps int64) error { // 1. Check if TAP already exists if _, err := netlink.LinkByName(tapName); err == nil { // TAP already exists, delete it first @@ -479,11 +488,254 @@ func (m *manager) createTAPDevice(tapName, bridgeName string, isolated bool) err } } + // 6. Apply download rate limiting (TBF on TAP egress) + if downloadBps > 0 { + if err := m.applyDownloadRateLimit(tapName, downloadBps); err != nil { + return fmt.Errorf("apply download rate limit: %w", err) + } + } + + // 7. Apply upload rate limiting (HTB class on bridge) + if uploadBps > 0 { + if err := m.addVMClass(bridgeName, tapName, uploadBps, uploadCeilBps); err != nil { + return fmt.Errorf("apply upload rate limit: %w", err) + } + } + + return nil +} + +// applyDownloadRateLimit applies download (external→VM) rate limiting using TBF on TAP egress. +func (m *manager) applyDownloadRateLimit(tapName string, rateLimitBps int64) error { + rateStr := formatTcRate(rateLimitBps) + + // Use Token Bucket Filter (tbf) for download shaping + // burst: bucket size = (rate * multiplier) / 250 for HZ=250 kernels + // The multiplier allows initial burst before settling to sustained rate. + // latency: max time a packet can wait in queue + multiplier := m.GetDownloadBurstMultiplier() + burstBytes := (rateLimitBps * int64(multiplier)) / 250 + if burstBytes < 1540 { + burstBytes = 1540 // Minimum burst for standard MTU + } + + cmd := exec.Command("tc", "qdisc", "add", "dev", tapName, "root", "tbf", + "rate", rateStr, + "burst", fmt.Sprintf("%d", burstBytes), + "latency", "50ms") + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("tc qdisc add tbf: %w (output: %s)", err, string(output)) + } + + return nil +} + +// removeRateLimit removes any rate limiting from a TAP device. +func (m *manager) removeRateLimit(tapName string) error { + cmd := exec.Command("tc", "qdisc", "del", "dev", tapName, "root") + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + // Ignore errors - qdisc may not exist + cmd.Run() + return nil +} + +// setupBridgeHTB sets up HTB qdisc on bridge for upload (VM→external) fair sharing. +// This is one-time setup - per-VM classes are added dynamically via addVMClass. +func (m *manager) setupBridgeHTB(ctx context.Context, bridgeName string, capacityBps int64) error { + log := logger.FromContext(ctx) + + if capacityBps <= 0 { + log.DebugContext(ctx, "skipping HTB setup - no capacity configured", "bridge", bridgeName) + return nil + } + + // Check if HTB qdisc already exists + checkCmd := exec.Command("tc", "qdisc", "show", "dev", bridgeName) + checkCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + output, err := checkCmd.Output() + if err == nil && strings.Contains(string(output), "htb") { + log.InfoContext(ctx, "HTB qdisc ready", "bridge", bridgeName, "status", "existing") + return nil + } + + rateStr := formatTcRate(capacityBps) + + // 1. Add root HTB qdisc (no default - all traffic must be classified) + cmd := exec.Command("tc", "qdisc", "add", "dev", bridgeName, "root", + "handle", htbRootHandle, "htb") + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("tc qdisc add htb: %w (output: %s)", err, string(output)) + } + + // 2. Add root class for total capacity + cmd = exec.Command("tc", "class", "add", "dev", bridgeName, "parent", htbRootHandle, + "classid", htbRootClassID, "htb", "rate", rateStr) + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("tc class add root: %w (output: %s)", err, string(output)) + } + + log.InfoContext(ctx, "HTB qdisc ready", "bridge", bridgeName, "capacity", rateStr, "status", "configured") + return nil +} + +// addVMClass adds an HTB class for a VM on the bridge for upload rate limiting. +// Called during TAP device creation. rateBps is guaranteed, ceilBps is burst ceiling. +func (m *manager) addVMClass(bridgeName, tapName string, rateBps, ceilBps int64) error { + if rateBps <= 0 { + return nil // No rate limiting configured + } + + // Use first 4 hex chars of TAP name suffix as class ID (e.g., "hype-a1b2c3d4" → "a1b2") + // This ensures unique, stable class IDs per VM + classID := deriveClassID(tapName) + fullClassID := fmt.Sprintf("1:%s", classID) + + rateStr := formatTcRate(rateBps) + if ceilBps <= 0 { + ceilBps = rateBps + } + ceilStr := formatTcRate(ceilBps) + + // 1. Add HTB class for this VM + cmd := exec.Command("tc", "class", "add", "dev", bridgeName, "parent", htbRootClassID, + "classid", fullClassID, "htb", "rate", rateStr, "ceil", ceilStr, "prio", "1") + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("tc class add vm: %w (output: %s)", err, string(output)) + } + + // 2. Add fq_codel to this class for better latency under load + cmd = exec.Command("tc", "qdisc", "add", "dev", bridgeName, "parent", fullClassID, "fq_codel") + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + // Ignore errors - fq_codel may not be available + cmd.Run() + + // 3. Add filter to classify traffic from this TAP to this class + // Use basic match on incoming interface (rt_iif) + tapLink, err := netlink.LinkByName(tapName) + if err != nil { + return fmt.Errorf("get TAP link for filter: %w", err) + } + tapIndex := tapLink.Attrs().Index + + cmd = exec.Command("tc", "filter", "add", "dev", bridgeName, "parent", htbRootHandle, + "protocol", "all", "prio", "1", "basic", + "match", fmt.Sprintf("meta(rt_iif eq %d)", tapIndex), + "flowid", fullClassID) + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("tc filter add: %w (output: %s)", err, string(output)) + } + + return nil +} + +// removeVMClass removes the HTB class for a VM from the bridge. +func (m *manager) removeVMClass(bridgeName, tapName string) error { + classID := deriveClassID(tapName) + fullClassID := fmt.Sprintf("1:%s", classID) + + // Delete filter first (by matching flowid) + // List filters and delete matching ones + listCmd := exec.Command("tc", "filter", "show", "dev", bridgeName, "parent", htbRootHandle) + listCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + output, _ := listCmd.Output() + + // Parse filter output to find handle for this flowid + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, fullClassID) && strings.Contains(line, "filter") { + // Extract filter handle (e.g., "filter parent 1: protocol all pref 1 basic chain 0 handle 0x1") + fields := strings.Fields(line) + for i, f := range fields { + if f == "handle" && i+1 < len(fields) { + handle := fields[i+1] + delCmd := exec.Command("tc", "filter", "del", "dev", bridgeName, "parent", htbRootHandle, + "protocol", "all", "prio", "1", "handle", handle, "basic") + delCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + delCmd.Run() // Best effort + break + } + } + } + } + + // Delete child qdisc (fq_codel) before deleting the class + qdiscCmd := exec.Command("tc", "qdisc", "del", "dev", bridgeName, "parent", fullClassID) + qdiscCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + qdiscCmd.Run() // Best effort - may not exist + + // Delete the class + cmd := exec.Command("tc", "class", "del", "dev", bridgeName, "classid", fullClassID) + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + // Ignore errors - class may not exist + cmd.Run() + return nil } -// deleteTAPDevice removes TAP device +// deriveClassID derives a unique HTB class ID from a TAP name. +// Uses first 4 hex characters after the prefix (e.g., "hype-a1b2c3d4" → "a1b2"). +func deriveClassID(tapName string) string { + // Hash the TAP name to get a valid hex class ID. + // tc class IDs must be hexadecimal (0-9, a-f), but CUID2 instance IDs + // use base-36 (0-9, a-z) which includes invalid chars like t, w, v, etc. + // Using FNV-1a for speed. Limited to 16 bits since tc class IDs max at 0xFFFF. + h := fnv.New32a() + h.Write([]byte(tapName)) + hash := h.Sum32() + // Use only 16 bits (tc class ID max is 0xFFFF) + return fmt.Sprintf("%04x", hash&0xFFFF) +} + +// formatTcRate formats bytes per second as a tc rate string. +func formatTcRate(bytesPerSec int64) string { + bitsPerSec := bytesPerSec * 8 + switch { + case bitsPerSec >= 1000000000: + return fmt.Sprintf("%dgbit", bitsPerSec/1000000000) + case bitsPerSec >= 1000000: + return fmt.Sprintf("%dmbit", bitsPerSec/1000000) + case bitsPerSec >= 1000: + return fmt.Sprintf("%dkbit", bitsPerSec/1000) + default: + return fmt.Sprintf("%dbit", bitsPerSec) + } +} + +// deleteTAPDevice removes TAP device and its associated HTB class on the bridge. func (m *manager) deleteTAPDevice(tapName string) error { + // Remove HTB class from bridge before deleting TAP + m.removeVMClass(m.config.BridgeName, tapName) + link, err := netlink.LinkByName(tapName) if err != nil { // TAP doesn't exist, nothing to do @@ -585,3 +837,118 @@ func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs [] return deleted } + +// CleanupOrphanedClasses removes HTB classes on the bridge that don't have matching TAP devices. +// This handles the case where a TAP was deleted externally (manual deletion, reboot, etc.) +// but the HTB class persists on the bridge. +// Returns the number of classes deleted. +func (m *manager) CleanupOrphanedClasses(ctx context.Context) int { + log := logger.FromContext(ctx) + bridgeName := m.config.BridgeName + + // List all HTB classes on the bridge + cmd := exec.Command("tc", "class", "show", "dev", bridgeName) + output, err := cmd.Output() + if err != nil { + log.DebugContext(ctx, "no HTB classes to clean up", "bridge", bridgeName) + return 0 + } + + // Build set of class IDs that belong to existing TAP devices + validClassIDs := make(map[string]bool) + links, err := netlink.LinkList() + if err == nil { + for _, link := range links { + name := link.Attrs().Name + if strings.HasPrefix(name, TAPPrefix) { + classID := deriveClassID(name) + validClassIDs[classID] = true + } + } + } + + // Parse class output and find orphaned classes + // Format: "class htb 1:xxxx parent 1:1 ..." + deleted := 0 + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if !strings.Contains(line, "class htb 1:") { + continue + } + + // Extract class ID (e.g., "1:a3f2") + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + fullClassID := fields[2] // "1:xxxx" + + // Skip root class + if fullClassID == htbRootClassID { + continue + } + + // Extract just the minor part (after "1:") + parts := strings.Split(fullClassID, ":") + if len(parts) != 2 { + continue + } + classID := parts[1] + + // Check if this class belongs to an existing TAP + if validClassIDs[classID] { + continue + } + + // Orphaned class - delete it with warning + log.WarnContext(ctx, "cleaning up orphaned HTB class", "class", fullClassID, "bridge", bridgeName) + + // Delete filter first (find and delete by flowid) + // Filters are created with 'basic' classifier, format: "handle 0xN flowid 1:xxxx" + filterCmd := exec.Command("tc", "filter", "show", "dev", bridgeName) + filterCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if filterOutput, err := filterCmd.Output(); err == nil { + filterLines := strings.Split(string(filterOutput), "\n") + for _, fline := range filterLines { + if strings.Contains(fline, fullClassID) { + // Extract filter handle (format: "handle 0x2 flowid 1:ffd") + ffields := strings.Fields(fline) + for i, f := range ffields { + if f == "handle" && i+1 < len(ffields) { + handle := ffields[i+1] + // Use 'basic' classifier (not u32) to match how filters were created + delCmd := exec.Command("tc", "filter", "del", "dev", bridgeName, "parent", "1:", "handle", handle, "prio", "1", "basic") + delCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + delCmd.Run() // Best effort + break + } + } + } + } + } + + // Delete child qdisc (fq_codel) before deleting the class + delQdiscCmd := exec.Command("tc", "qdisc", "del", "dev", bridgeName, "parent", fullClassID) + delQdiscCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + delQdiscCmd.Run() // Best effort - may not exist + + // Delete the class + delClassCmd := exec.Command("tc", "class", "del", "dev", bridgeName, "classid", fullClassID) + delClassCmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + if output, err := delClassCmd.CombinedOutput(); err != nil { + log.WarnContext(ctx, "failed to delete orphaned class", "class", fullClassID, "error", err, "output", string(output)) + continue + } + deleted++ + } + + return deleted +} diff --git a/lib/network/manager.go b/lib/network/manager.go index a48d334..12e5d98 100644 --- a/lib/network/manager.go +++ b/lib/network/manager.go @@ -19,13 +19,23 @@ type Manager interface { // Instance allocation operations (called by instance manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*NetworkConfig, error) - RecreateAllocation(ctx context.Context, instanceID string) error + RecreateAllocation(ctx context.Context, instanceID string, downloadBps, uploadBps int64) error ReleaseAllocation(ctx context.Context, alloc *Allocation) error + // SetupHTB initializes HTB qdisc on the bridge for upload fair sharing. + // Should be called during network initialization with the total network capacity. + SetupHTB(ctx context.Context, capacityBps int64) error + // Queries (derive from CH/snapshots) GetAllocation(ctx context.Context, instanceID string) (*Allocation, error) ListAllocations(ctx context.Context) ([]Allocation, error) NameExists(ctx context.Context, name string) (bool, error) + + // GetUploadBurstMultiplier returns the configured multiplier for upload burst ceiling. + GetUploadBurstMultiplier() int + + // GetDownloadBurstMultiplier returns the configured multiplier for download burst bucket. + GetDownloadBurstMultiplier() int } // manager implements the Manager interface @@ -91,6 +101,11 @@ func (m *manager) Initialize(ctx context.Context, runningInstanceIDs []string) e log.InfoContext(ctx, "cleaned up orphaned TAP devices", "count", deleted) } + // Cleanup orphaned HTB classes (TAPs deleted externally but classes remain) + if deleted := m.CleanupOrphanedClasses(ctx); deleted > 0 { + log.InfoContext(ctx, "cleaned up orphaned HTB classes", "count", deleted) + } + log.InfoContext(ctx, "network manager initialized") return nil } @@ -113,3 +128,27 @@ func (m *manager) getDefaultNetwork(ctx context.Context) (*Network, error) { CreatedAt: time.Time{}, // Unknown for default }, nil } + +// SetupHTB initializes HTB qdisc on the bridge for upload fair sharing. +// capacityBps is the total network capacity in bytes per second. +func (m *manager) SetupHTB(ctx context.Context, capacityBps int64) error { + return m.setupBridgeHTB(ctx, m.config.BridgeName, capacityBps) +} + +// GetUploadBurstMultiplier returns the configured multiplier for upload burst ceiling. +// Defaults to 4 if not configured. +func (m *manager) GetUploadBurstMultiplier() int { + if m.config.UploadBurstMultiplier < 1 { + return DefaultUploadBurstMultiplier + } + return m.config.UploadBurstMultiplier +} + +// GetDownloadBurstMultiplier returns the configured multiplier for download burst bucket. +// Defaults to 4 if not configured. +func (m *manager) GetDownloadBurstMultiplier() int { + if m.config.DownloadBurstMultiplier < 1 { + return DefaultDownloadBurstMultiplier + } + return m.config.DownloadBurstMultiplier +} diff --git a/lib/network/types.go b/lib/network/types.go index e63d31b..5b55ace 100644 --- a/lib/network/types.go +++ b/lib/network/types.go @@ -2,14 +2,25 @@ package network import "time" +// DefaultUploadBurstMultiplier is the default multiplier applied to guaranteed upload rate +// to calculate the burst ceiling. This allows VMs to burst above their guaranteed rate +// when other VMs are not using their full bandwidth allocation. +// Configurable via UPLOAD_BURST_MULTIPLIER environment variable. +const DefaultUploadBurstMultiplier = 4 + +// DefaultDownloadBurstMultiplier is the default multiplier for the TBF burst bucket size. +// A larger bucket allows faster initial burst before settling to sustained rate. +// Configurable via DOWNLOAD_BURST_MULTIPLIER environment variable. +const DefaultDownloadBurstMultiplier = 4 + // Network represents a virtual network for instances type Network struct { - Name string // "default", "internal" - Subnet string // "192.168.0.0/16" - Gateway string // "192.168.0.1" - Bridge string // "vmbr0" (derived from kernel) - Isolated bool // Bridge_slave isolation mode - Default bool // True for default network + Name string // "default", "internal" + Subnet string // "192.168.0.0/16" + Gateway string // "192.168.0.1" + Bridge string // "vmbr0" (derived from kernel) + Isolated bool // Bridge_slave isolation mode + Default bool // True for default network CreatedAt time.Time } @@ -39,7 +50,9 @@ type NetworkConfig struct { // AllocateRequest is the request to allocate network for an instance // Always allocates from the default network type AllocateRequest struct { - InstanceID string - InstanceName string + InstanceID string + InstanceName string + DownloadBps int64 // Download rate limit in bytes/sec (external→VM, TAP egress TBF) + UploadBps int64 // Upload rate limit in bytes/sec (VM→external, HTB class rate) + UploadCeilBps int64 // Upload ceiling in bytes/sec (HTB burst when bandwidth available, 0 = same as UploadBps) } - diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 129eaf9..450b903 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -141,6 +141,9 @@ type CreateInstanceRequest struct { // Devices Device IDs or names to attach for GPU/PCI passthrough Devices *[]string `json:"devices,omitempty"` + // DiskIoBps Disk I/O rate limit (e.g., "100MB/s", "500MB/s"). Defaults to proportional share based on CPU allocation if configured. + DiskIoBps *string `json:"disk_io_bps,omitempty"` + // Env Environment variables Env *map[string]string `json:"env,omitempty"` @@ -158,6 +161,12 @@ type CreateInstanceRequest struct { // Network Network configuration for the instance Network *struct { + // BandwidthDownload Download bandwidth limit (external→VM, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + + // BandwidthUpload Upload bandwidth limit (VM→external, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + // Enabled Whether to attach instance to the default network Enabled *bool `json:"enabled,omitempty"` } `json:"network,omitempty"` @@ -228,6 +237,21 @@ type Device struct { // DeviceType Type of PCI device type DeviceType string +// DiskBreakdown defines model for DiskBreakdown. +type DiskBreakdown struct { + // ImagesBytes Disk used by exported rootfs images + ImagesBytes *int64 `json:"images_bytes,omitempty"` + + // OciCacheBytes Disk used by OCI layer cache (shared blobs) + OciCacheBytes *int64 `json:"oci_cache_bytes,omitempty"` + + // OverlaysBytes Disk used by instance overlays (rootfs + volume overlays) + OverlaysBytes *int64 `json:"overlays_bytes,omitempty"` + + // VolumesBytes Disk used by volumes + VolumesBytes *int64 `json:"volumes_bytes,omitempty"` +} + // Error defines model for Error. type Error struct { // Code Application-specific error code (machine-readable) @@ -357,6 +381,9 @@ type Instance struct { // CreatedAt Creation timestamp (RFC3339) CreatedAt time.Time `json:"created_at"` + // DiskIoBps Disk I/O rate limit (human-readable, e.g., "100MB/s") + DiskIoBps *string `json:"disk_io_bps,omitempty"` + // Env Environment variables Env *map[string]string `json:"env,omitempty"` @@ -380,6 +407,12 @@ type Instance struct { // Network Network configuration of the instance Network *struct { + // BandwidthDownload Download bandwidth limit (human-readable, e.g., "1Gbps", "125MB/s") + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + + // BandwidthUpload Upload bandwidth limit (human-readable, e.g., "1Gbps", "125MB/s") + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + // Enabled Whether instance is attached to the default network Enabled *bool `json:"enabled,omitempty"` @@ -465,6 +498,64 @@ type PathInfo struct { Size *int64 `json:"size,omitempty"` } +// ResourceAllocation defines model for ResourceAllocation. +type ResourceAllocation struct { + // Cpu vCPUs allocated + Cpu *int `json:"cpu,omitempty"` + + // DiskBytes Disk allocated in bytes (overlay + volumes) + DiskBytes *int64 `json:"disk_bytes,omitempty"` + + // InstanceId Instance identifier + InstanceId *string `json:"instance_id,omitempty"` + + // InstanceName Instance name + InstanceName *string `json:"instance_name,omitempty"` + + // MemoryBytes Memory allocated in bytes + MemoryBytes *int64 `json:"memory_bytes,omitempty"` + + // NetworkDownloadBps Download bandwidth limit in bytes/sec (external→VM) + NetworkDownloadBps *int64 `json:"network_download_bps,omitempty"` + + // NetworkUploadBps Upload bandwidth limit in bytes/sec (VM→external) + NetworkUploadBps *int64 `json:"network_upload_bps,omitempty"` +} + +// ResourceStatus defines model for ResourceStatus. +type ResourceStatus struct { + // Allocated Currently allocated resources + Allocated int64 `json:"allocated"` + + // Available Available for allocation (effective_limit - allocated) + Available int64 `json:"available"` + + // Capacity Raw host capacity + Capacity int64 `json:"capacity"` + + // EffectiveLimit Capacity after oversubscription (capacity * ratio) + EffectiveLimit int64 `json:"effective_limit"` + + // OversubRatio Oversubscription ratio applied + OversubRatio float64 `json:"oversub_ratio"` + + // Source How capacity was determined (detected, configured) + Source *string `json:"source,omitempty"` + + // Type Resource type + Type string `json:"type"` +} + +// Resources defines model for Resources. +type Resources struct { + Allocations []ResourceAllocation `json:"allocations"` + Cpu ResourceStatus `json:"cpu"` + Disk ResourceStatus `json:"disk"` + DiskBreakdown *DiskBreakdown `json:"disk_breakdown,omitempty"` + Memory ResourceStatus `json:"memory"` + Network ResourceStatus `json:"network"` +} + // Volume defines model for Volume. type Volume struct { // Attachments List of current attachments (empty if not attached) @@ -737,6 +828,9 @@ type ClientInterface interface { AttachVolume(ctx context.Context, id string, volumeId string, body AttachVolumeJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetResources request + GetResources(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListVolumes request ListVolumes(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1124,6 +1218,18 @@ func (c *Client) AttachVolume(ctx context.Context, id string, volumeId string, b return c.Client.Do(req) } +func (c *Client) GetResources(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetResourcesRequest(c.Server) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) ListVolumes(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewListVolumesRequest(c.Server) if err != nil { @@ -2165,6 +2271,33 @@ func NewAttachVolumeRequestWithBody(server string, id string, volumeId string, c return req, nil } +// NewGetResourcesRequest generates requests for GetResources +func NewGetResourcesRequest(server string) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/resources") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewListVolumesRequest generates requests for ListVolumes func NewListVolumesRequest(server string) (*http.Request, error) { var err error @@ -2431,6 +2564,9 @@ type ClientWithResponsesInterface interface { AttachVolumeWithResponse(ctx context.Context, id string, volumeId string, body AttachVolumeJSONRequestBody, reqEditors ...RequestEditorFn) (*AttachVolumeResponse, error) + // GetResourcesWithResponse request + GetResourcesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetResourcesResponse, error) + // ListVolumesWithResponse request ListVolumesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListVolumesResponse, error) @@ -3080,6 +3216,29 @@ func (r AttachVolumeResponse) StatusCode() int { return 0 } +type GetResourcesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Resources + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetResourcesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetResourcesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type ListVolumesResponse struct { Body []byte HTTPResponse *http.Response @@ -3452,6 +3611,15 @@ func (c *ClientWithResponses) AttachVolumeWithResponse(ctx context.Context, id s return ParseAttachVolumeResponse(rsp) } +// GetResourcesWithResponse request returning *GetResourcesResponse +func (c *ClientWithResponses) GetResourcesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetResourcesResponse, error) { + rsp, err := c.GetResources(ctx, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetResourcesResponse(rsp) +} + // ListVolumesWithResponse request returning *ListVolumesResponse func (c *ClientWithResponses) ListVolumesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListVolumesResponse, error) { rsp, err := c.ListVolumes(ctx, reqEditors...) @@ -4606,6 +4774,39 @@ func ParseAttachVolumeResponse(rsp *http.Response) (*AttachVolumeResponse, error return response, nil } +// ParseGetResourcesResponse parses an HTTP response from a GetResourcesWithResponse call +func ParseGetResourcesResponse(rsp *http.Response) (*GetResourcesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetResourcesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Resources + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseListVolumesResponse parses an HTTP response from a ListVolumesWithResponse call func ParseListVolumesResponse(rsp *http.Response) (*ListVolumesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -4860,6 +5061,9 @@ type ServerInterface interface { // Attach volume to instance // (POST /instances/{id}/volumes/{volumeId}) AttachVolume(w http.ResponseWriter, r *http.Request, id string, volumeId string) + // Get host resource capacity and allocations + // (GET /resources) + GetResources(w http.ResponseWriter, r *http.Request) // List volumes // (GET /volumes) ListVolumes(w http.ResponseWriter, r *http.Request) @@ -5034,6 +5238,12 @@ func (_ Unimplemented) AttachVolume(w http.ResponseWriter, r *http.Request, id s w.WriteHeader(http.StatusNotImplemented) } +// Get host resource capacity and allocations +// (GET /resources) +func (_ Unimplemented) GetResources(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + // List volumes // (GET /volumes) func (_ Unimplemented) ListVolumes(w http.ResponseWriter, r *http.Request) { @@ -5828,6 +6038,26 @@ func (siw *ServerInterfaceWrapper) AttachVolume(w http.ResponseWriter, r *http.R handler.ServeHTTP(w, r) } +// GetResources operation middleware +func (siw *ServerInterfaceWrapper) GetResources(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetResources(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ListVolumes operation middleware func (siw *ServerInterfaceWrapper) ListVolumes(w http.ResponseWriter, r *http.Request) { @@ -6121,6 +6351,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/volumes/{volumeId}", wrapper.AttachVolume) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/resources", wrapper.GetResources) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/volumes", wrapper.ListVolumes) }) @@ -7187,6 +7420,31 @@ func (response AttachVolume500JSONResponse) VisitAttachVolumeResponse(w http.Res return json.NewEncoder(w).Encode(response) } +type GetResourcesRequestObject struct { +} + +type GetResourcesResponseObject interface { + VisitGetResourcesResponse(w http.ResponseWriter) error +} + +type GetResources200JSONResponse Resources + +func (response GetResources200JSONResponse) VisitGetResourcesResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetResources500JSONResponse Error + +func (response GetResources500JSONResponse) VisitGetResourcesResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type ListVolumesRequestObject struct { } @@ -7433,6 +7691,9 @@ type StrictServerInterface interface { // Attach volume to instance // (POST /instances/{id}/volumes/{volumeId}) AttachVolume(ctx context.Context, request AttachVolumeRequestObject) (AttachVolumeResponseObject, error) + // Get host resource capacity and allocations + // (GET /resources) + GetResources(ctx context.Context, request GetResourcesRequestObject) (GetResourcesResponseObject, error) // List volumes // (GET /volumes) ListVolumes(ctx context.Context, request ListVolumesRequestObject) (ListVolumesResponseObject, error) @@ -8171,6 +8432,30 @@ func (sh *strictHandler) AttachVolume(w http.ResponseWriter, r *http.Request, id } } +// GetResources operation middleware +func (sh *strictHandler) GetResources(w http.ResponseWriter, r *http.Request) { + var request GetResourcesRequestObject + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetResources(ctx, request.(GetResourcesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetResources") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetResourcesResponseObject); ok { + if err := validResponse.VisitGetResourcesResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ListVolumes operation middleware func (sh *strictHandler) ListVolumes(w http.ResponseWriter, r *http.Request) { var request ListVolumesRequestObject @@ -8292,112 +8577,126 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+w9DXMTObJ/RTXvrs55ZztO+Djw1dWrbAKsrwikCGTf3YYX5Jm2rWVGGiSNE0Plv79S", - "S5ovj+0JEEMOqrZqTUYjtVr9re6eT0EoklRw4FoFw0+BCmeQUPx5oDUNZ2cizhJ4BR8yUNr8OZUiBakZ", - "4KBEZFxfpFTPzL8iUKFkqWaCB8PghOoZuZyBBDLHWYiaiSyOyBgIvgdR0A3giiZpDMEw2E243o2opkE3", - "0IvU/Elpyfg0uO4GEmgkeLywy0xoFutgOKGxgm5t2WMzNaGKmFd6+E4+31iIGCgPrnHGDxmTEAXD38vb", - "eJsPFuM/INRm8YM5ZTEdx3AEcxbCMhrCTErg+iKSbA5yGRWH9nm8IGOR8YjYcaTDszgmbEK44LBTQQaf", - "s4gZTJghZulgqGUGDZiJEKYLFjWcwOGI2MdkdEQ6M7iqLrL/t/GjYPWUnCawPOmvWUJ5zyDXgOXnx7Hl", - "uZ/fb5qZiSTJLqZSZOnyzKOXx8dvCD4kPEvGIMszPtrP52NcwxSkmTAN2QWNIglKNe/fPyzDNhgMBkO6", - "PxwM+oMmKOfAIyFXotQ+bkbp3iCCNVO2QqmbfwmlL85GR6MDcihkKiTFd5dWqhF2GT3lfZXJpnoqTfR/", - "KIFqR/wrRUHz1l7iDxqTaSzGNI4XJOPsQ1ahmz4ZGRbQJJViziKIuoTiA8IUoZkWvSlwkFRDRCZSJETP", - "gJTOlnSgP+13ybnZbs8cbo/u9waD3uA8qJ5OfL83TbOgG6RUa5AGwP/7nfY+HvT+Peg9flv8vOj33v71", - "T00H2ZbgiJggnG6fHX8qXeKBLVNhHdD1FLrmkFcf3yih0xuf3uGIMPMekTABCdzsxMIfifA9yD4TuzEb", - "SyoXu3zK+NUwphqUru5m/diN+0PY1myMT83Wb7i1Gs8huXVicQkypApIDIZAVJdEbMq06hJqxDZVM1DE", - "6JS/k5ByQ7NKU6mJkAR4RC6ZnhGK46oYSBY9mrIes6AG3SChV8+BT43efHhviR4NMXbcj97b//Z/2vmf", - "RpKUWQwNxPhKZJrxKcHHZCIk0TOmSAED05Dge3+SMAmGwX/tFsbArrMEdj12sxjMWgnjI/vaXg4JlZIu", - "mk/NA7fu9JSmfI1csQzUsL8jr9kUcdJSES0IRbsF9/vs5M2uYcmUKqVnUmTTWflUfvfy4G0JF0vYrW6y", - "GwCfm3E0ipgVbScVcBuUaRnoJ3zOpOAJcE3mVDJDfBXl9Cl48fLoycWTF2fB0GAiykIn6U9evnodDIN7", - "g8GgBFeBz5nQaZxNLxT7CBUzKbj37JegDshBDj9JIBFygRhzc5DOrMoeEyETqknM3gM5N/OdB0aE7T2r", - "C659XGoJCbNFCnLOlGgwjn7Nn5njyxSUadUSR58c2c3gCSuQxnoKBZ+waWY1Yd9AwbPEnGoYiyzqlZbs", - "Bh8gwWMuAG0YtGytGNHXSipuEHc0ThmHlfKu+73IqEsh38eCRr29ryyiOGgz9/IWX9gH1cN0BAD5+RuN", - "XeEy4AYjUYXMrX1cnf63GegZyJJc8FOaP1n9jK8TD2EJIxWDu+w6LLGemIOM6aKB9fYGDbz3m2QaT9S9", - "RyKm3hPz8gbGM7NZznswWGa9QTPvNQDVANMvhqKcJGgDSQ7I3v6x+7nfVhrMwzRTFZD26+C8QPvfGFFz", - "JnVGY3J48qYiKBvdAetoNigL68eWFYQ7/5weqCah0UiG/jRD3dVKQdqZ0etcVhfNOtHKldU6cYPT3eSX", - "5HZ2mCktEsIi4JpNmPEyayY0qxrb1RObi7hnfHCUAC3FlAV32V9JFnYqeyirSPNiOl6e8tRQIONkyqZ0", - "vNBVFbk3WD76ZkT7+ZtQvcqXt+QB0YUWDS6qp5bRkcGjH9vGT0fP/0KLi/mENcycS6rCZ2CKhLXAgSNa", - "M0UvDZkLJHTJ5YwZ2aaIRwKK0LPjsunTP+c9YoAbkqN8gXzafEqjRAzTW4OgI2QJCMZRP48XO4SSs+M+", - "eZ1D+xdFONVsDj64MaOKjAE4ybhRKRDh+hiyKQOQKWOjMl1/3bkZNg6ygxaecM/6xJgMCeXkksUxeogJ", - "1SxE93LMavu5nAF3B2VWMgKAF4bFOS9Tlgso1UV+N0DJANEF1Q12NkyZ0rKQHErTJCWdV08P792797gu", - "pPcf9AZ7vb0Hr/cGw4H5799BN7DC1dgOVEPPiZ9thHqa5jqoygvnsJclyuGb0dG+0wjVdfTH+/Txo6sr", - "qh8/ZJfq8cdkLKd/3KNbCQY1i6ejItJAOpkC2fOiz1BVU3yh5MaviB98dljgRnEo+4f16sfu7rUZeRuR", - "q5pcxXARDul+RmypLgQrfLVaRr92aKjux/zV2AcF5ZdcABflCVlp2gKvT6S0jkgtiiuihnUO0jRmIXJ3", - "T6UQsgkLCZgZiHmBdBKULJBbSlW0jml0IZ0mb2RpTVncQDMl/8wu5kaSjhHLSRZrlsZgnyGVtjJWcOdH", - "OFOTb8s4B3kBHj03mCkBpRqdpZoP4/eSD0EtE8E4m04NSsqoO2YKlUOh0xjE0dD6XhtJFU+zAKyJvMp7", - "aEkNz4331YthDnGZCKxEMcAmQgLJ6cQeWmVXjM9pzKILxtOskSRWovJpJtFEsJMSOhaZRnPAHlh5EQz2", - "oZk3MRzXiKwldPwKNLYXOVVMKE115twuy17ifdWZFu83HoebpOkYRt7Nrh1A0iDFDo+PrIwOBdeUcZAk", - "AU3dtVEptIMRxqAb9AxNRRQSwYmYTP6+PtizworLGWSdHXBY9h5uzwZgU+cU1K0QJeI5RCShnE1AaeJG", - "lldWM7r/4OGQjsO9/XsRTO4/eNjv95uWAa7lIhWMNyz1JH/W7ih2bUCkV8zZV7MvO4dbCMK12cun4OTg", - "9a/BMNjNlNyNRUjjXTVmfFj6d/7P4gH+sP8cM94YvMtlbg1SFDFOIhiPw7KRMZwnlMW1G9Q0i2P396HZ", - "CYcwJ0iBwmajl9JsQr0wpBmzjxCRxisBTafGlrIU92Wx/27wIYMMLlKhmF19yZBxT4w3Ms5YHBF8o3yb", - "qu2fqr7t/srtl0xI9Bitx7lsSOZRGrOyGePWzLhmsXWaKis+uPfw0d8Gj/f2S8zNuH54P2gFSi52a5Ea", - "3LN7Wpg8KfDIalBDBvZXKPjccAX+A+EzcsYSTkWA+2dLh3Ep5HvGpxcRa6DO3+xDEjEJocYY8mYeCnZp", - "mm4mxWaHPpdp+fY3WJDu+qJBu3xzSf45rld19ZfTf374X3Xytz/2Pjw/O/vX/Nk/j16wf53FJy+/KMa8", - "/ubqm14/rY2uob9RuXZqSx7HVIcNhs9MKL0Ca+4J0YIk5uU+OaScjGF4znvkOdMgaTwk5wFNWd8hsx+K", - "5DwgHbiiobZvEcGJmYrMgEYgd8zLJzbObl7+5MMU1/U5ogWnCQuJdEgeU2XcWU5UNo5EQhnfOefn3M1F", - "/EYUhm/Mr4iENNWZBHMiJMxkvCBjSUPIb9OLxbvkE03T651zrmdUE7jS0uwgpVLn19x+BTxoB5UND7nh", - "EJE5jTNQJEREnfNcf0QGBDOJpnIKup+HZNHer4VoViCl0ScXUleizI8G3YZzJGacOciYKQ2c5PcPTCHx", - "ko6/I3g0qLD/o8GjzZHInIbWkB9S93JulSfKFvxhCRiXtsL4YqZ1ujlZCuWN5RHy6+vXJwYN5v+nxE9U", - "4CI/4o7g8YJQ4xeDsvE1HaNN4q5ldoKmGJo93ZYbem0Hm9ditXkfT3Bh8vr5KdEgE8at/O6EBp0T476D", - "jfQwpTJDioySg8PjJzv9FslhiNsc/jXn+DrfYS1g76+xloMY+EZxCWHw2yWjo64xpxyHFoYWRlCfCkli", - "K2AKvh6SNwqq9xl4VDbYY08yXhSJMlaqnwc7fsa0LimG5FVu39EclDz9piAGP2XBlzjtOf/NEIYN7y7N", - "3q3CioFr57840YbBXKqJi52gKl4tCtazfwPGkecFr98y3oy3y9eTZrFm0ijO/tYtkHs3s0BuJ5VhOTGB", - "qgvFaapmQq+++KDEjyFwxZRWy2kArUL1y2kQVYFvExzW3HR+zYQGmXGOtw71bXz1VIVvGcf//tIk1iY2", - "fGl2gjN52iUnNJF6We75K8TPzkfoBqzh+uRAKTblEJHRSZEmWTjIfvraFcDj/f7ew0f9vcGgvzdoEy5I", - "aLhm7eODw/aLD/atAzWk42EYDWHyBeEKd2xWQdH4ki4UOfcmxHlgbZaSsVIiSmdmtAqYLqd9fF6WR/0i", - "ZlMex03yNlpJM0wQWqGKTjF56OZ66MFKPbTxVI1zD5sNRctEpzjYv3Vxk0AakFBkccT/osnYcJ41HSFy", - "Fq4CbSnFjmWKvOHvubjk1a3beIrh3w8ZyAU5Oz6uRN8kTDLVLkdAaZGmK89BpDc6hv0N5sBGaEppOttI", - "zalLwpJ8/eqJOOVQgb9OslTXImRQprvVGSI4HYYDbFJPNDSUQdzsZJxpkufmGZI7NFqelGwHmw+BFvor", - "a0aYGVBnhOZJvMjNi7Uvn1BDfv7dFP+1/o3TWaYjccnxHTXLNDH/QpDNFpx5tn4KS8lD8kLgOw7SrhH/", - "NTvPDqc8Gi+Wh9dtwo6NHhh3QQsJES7m2HJInuasmDOzY96OAvfTSgh37YhXqjvWrXAmmTutoBs4rAfd", - "wKIw6AYeM+an3SH+QuCDbuAAabx0PqF6NuITsewI3ERkufC+d7tSs0mljJiJgDOIdvrkZUV2ObzhhUGs", - "gEQZuBwYiwdJXdoRtc5QSvUMCRNfZHzar14x1BdsI0gsDOtznnBdN7CNzaOaQ9KvZYa4sga3IrQITrfy", - "Hpi6mLAY2kwsYZrFVBIc3w5ktUhixt+3mV0tkrGIWUjMC3WFNBFxLC4vzCP1D9zLTqvdmRcuijhMTcFY", - "4FwUzh5Ibd1iC/8wu9ypxfVDow127fu7WE3XxoRsvGd/ymIgCeZXvOHsqkTo1ZSc+/uDVdc4KyatXOBU", - "0wr37zdc1GyIATiSbdIUViGtyjBMfM1lLcGAKW10q0u6I6XBpANJqhf+isvry52bKciDfMKmDJCvHZQY", - "PP4a1yJv1t6D/IfkrJZtEr/IRmtk6UxXBh8bk8JGR3Vv1oogV7Bb9U9raSxK92wiQWMSy5rCYFuhi/LF", - "Bf6nWT1T4QbFwKs0SsE5Nr5YVANvEpQrwnw2oa20sxIkq8/GGqRfWDnNlC+Z/kyUOR90cyTdmj9G3vZy", - "kvAOrLGZLiXDi2+HIItYg4JcIyyrnfV+8jG9yldA4UwVqdUe2H0UsWasPtjpk1c+M4xN/BQIRtVm2Wt2", - "etuXlHuqWj6MdTXm3uVpZDwnf9ZItFW8VSPOYo3u+jJ2I7ogzCTTi1OjECwZjoFKkAeZJUPUFLgJ/HOx", - "ON4mXV9jiuCkIXv9GXCQLCQHJyOkkoRyOjVHdnZMYjaBcBHG4C4DltwGrEh6eTjq2VtMH6vDWDLTiBCf", - "8H1wMsJcU6nsuoP+fh+r4UQKnKYsGAb3+nuYTWvQgFvcLVUQOsPHMCKqslHkVO6RG2OQq1LBlR2/PxjY", - "XECunXClRTro7h/K5qdYBYvCto0edpUHyz7qUojdGwMSU8zBULrfzHU3uD/YuxFwG9M5m0B4w2mmZ0Ky", - "jxCZRR/cECOfteiIa5Ccxr7UD9zAgoSD4e9V4v397fXbbqCyJKFy4VHXjLdUqAYqKBe5B5bHQOlfRLT4", - "avttqqO/rjK0kV7XS0T49c7Z094yzl2afIEyS2JbOO1faJRf83Vcemx+fVnJxf9WRH9/cP/2Fy2VcOSJ", - "u0TYS1MLxOPbB+JQ8EnMQk16HhZXW01obKuCqgRyV8TBKwc1oX5fE7z1LgrDzXReVexS3/FlrdKo9YXZ", - "jvaoN6O5gRrJd1UqmvipSTaRzhFToTEuy9TSC2laan+jCj4tU9EnFl1bWykGGyuu0tAR/j1XOSmVNAEN", - "UiFMK7odkKI1DDMPfJwF3VzrRFbVSbeEw7ot+XaJYu+vLKDKeF03bEEoHtUE4jcUhLXL2lIV4V2i5jf5", - "Kfqqqetus4R7Bvr7Is3B9qwgX5b1Lcn8rlDUM9CeRXK0GSk4y8uJVpGXKzi6xYN2KzRs/NR4n5arLaD2", - "krDYln2VhDMI39sN4UXhejdyZIdsww6wVVM30P4O/J/qvoXjWOBqnbM4cjfHt+crVpp2tXIV978aBI7A", - "GpCMSV1jX4Zir6+pWvBw51v4jP/ZXmG9kvMOcdJJFsfYTsKVIRW1Y2V5uvvJ2Act7GTPbWttkTevnveA", - "hyKCyCUfrjZIfKnI17WW7YHZrfwkkzb+FaLKE8ZqY/QLzt/eHBQ9C/+8/9TlnP55/6nNOv3zvYOideHt", - "EMtgW6J529brHSY+Y7yyKtJQNNmCjk3WXj5qKwafq5y7icmXA/jT6mtj9ZXRtdbwy4sYb9H0q7Y13fI9", - "QU5sTdjGRz6T8Qcz+bYbenIUaa9IMTOjEot3yX3YuNPVY9kWXXeJ9VzGAcsprix/W8ZQC4Zcax140h0d", - "dV2pnS2QSyVM2NX2Iqoejq1biW7d7YdTD5Ixm2YiU+X6I6ysBFV0M6oI4LtmvxbqeaUF+x1T6WCbqmPr", - "BupPur8l07l+oFZ422uRTcazH7Ud47m4qmlvPXsIf1rPraznErrWW895Yc9tms/VvvJbt589vTUh3OVV", - "/ogW9B2zSil3Me7SZW9FxrU2UIti4fW6v2gwvPWL/nzx7dulvmnFXYwhYWUm9t73lmCha1abgt8bPQy2", - "K/u2bwLeZRJ7Vm4S02xsoSDajcW0bHbVS4gl0KRoREHMaEIVOUXAeqfANXkyN7vqn3PfUeadEpkM4R3J", - "CdV+hCOGULvG47HAxtoK58ea13c0Td/lnZN2huQZpneWsGsX7yiQjMYkFFyJ2NaOvpsnybvhcp742fEx", - "voRjZjYj/N0wbwae85gyo875OX8FOpNc4S5iqjR5QWLGQZGOOXAp4hgiMl6Qdwafpf3tYHsaM6NtixIv", - "zrl5g/EMlNsl41PC4dJNyCbknS3Kw/qDd7ZTzUquf25O6Rtxfnd1PbfdixZEIuJstx/A9rG4Lta3Fwu7", - "1rbFUnkVxd6gsdzp03KgC3HaiFI60dgqg2lDHyLTtl1uEyAW882grCzxWe7sOyWW0mukTNO0Lfk6MJGK", - "50myhoZJp+jnQpSORKb/qnQE0jaBc9S9irhJh4b2H5q+ty3LKh1mbEVzE6rsDptRFdjGjL4Q2v5rniSB", - "bXeT0KbC5haaRMOV3gUjVnoWrVWZWp9w2R8zJ4Mvks7p6ZOdnzqjpVmCKKsKe4fABs3hKuqxUq3ReXtl", - "B/zwlotvPfCNyXD7VxElKBg2ReHReOG+zJH397pTNQF4kMXOUN+5fTXyiH+2kkdcK4gfnkcK+vjBuSQU", - "ErtoKt+l6O4kb5U8jhK7d7CBTNGYpeu93rPj451VTGMbMa5kGfnTHXZ5lD+8TsGeOnePW2yTNJpvYF2w", - "0DCEXumje5+Vcdsuw7ga9ksfdLllAvaeUQulIbEO+ySLsbANs9bxw1oT/57NFejip7YM+dvPRZb6qpzz", - "MUyMPkxBmrXN62b+ku/R5Naeapqz74nlwe/Dr8UuCujKUb0Ka0uN6X0DhSbfKe/58NkgPUVHtdrbR5EO", - "flMRwZwrEpsfO2s9Xdv452b+7m1KuLy1VVNVq6XZnJh/BAk3qok136btzom1Z1BmFi9/8KCbxJpI16l5", - "kf7U8q5n3E+b+E7axHjRk++mM5U0RI2rXFfAZvvXNb7c/WR/jDZdF2oazs58+6nvQ5W6bjWblvEbvBNM", - "6fYUgfuE69Z5UuQNhe5o2QZ+RtltAUMn5YvPZi1gG5X9aNT99XNcyni8UYbLVnkr/zzy98Jb29Z8Dgaf", - "rl3Gx11hc0tpfida1FzbUlfnlZl+rsHzVvL8nGi5QZaf38HPhKgWOX4lZHkB39Q+UBGKN7l2eJ+cZmkq", - "pFZEXwrs6qrwZvWfpy9fkLGIFkOSv8eJbXLqCM51p3QfVIUIOwSad48xd5ZK/EBRUprAv5lK6KUizWJs", - "t40FFA7HVllRoqnsTz8SKsMZm0NDaKP8Nf5bTVasC/JukPjt7ZrtYU/S6qT1j9XmsFTPo7pHG89x3+ez", - "3yXJW0z6KUp9WseMU7lo26T1pcuWIGGmtEj8vKMj0ql9ZNs1sPWf4d75Hru5HtMrlmRJ3i742S+k4z5l", - "hl8txm8xs0lOU3AVAkQK7+F3btj5dbnpqzuLhi6OW81i9dJ0pYb/hhmsRas4c8T4KXdH5FoIElM5hZ0f", - "pk7M8VpRJjY6qhWJ3cHc27mnvsLOaJlt287BaGn330ambe58bjfP9uz7sYlL3bTuYLHXPDczVyX4fl8k", - "ONieSth2Yu/ZHY6hPANvUpeSenECM2MTwTwXIY1JBHOIRYq9ze3YoBtkMnadmoe79lvkM6E0fskwuH57", - "/f8BAAD//+6fgWjrmQAA", + "H4sIAAAAAAAC/+x9+3LbuNX4q2D4a6dyK8my7GQddTq/cewk606ceOLE+7XrfApEQhI2JMAAoGxtxv/2", + "AfqIfZJvcADwJlCmk1iJm3Q6s4pJAOeGc8PB4ccg5EnKGWFKBqOPgQznJMHw80ApHM7PeZwl5BX5kBGp", + "9J9TwVMiFCXwUsIzpsYpVnP9r4jIUNBUUc6CUXCK1RxdzokgaAGzIDnnWRyhCUEwjkRBNyBXOEljEoyC", + "7YSp7QgrHHQDtUz1n6QSlM2C624gCI44i5dmmSnOYhWMpjiWpFtb9kRPjbBEekgPxuTzTTiPCWbBNcz4", + "IaOCRMHo1zIab/OX+eQ3Eiq9+MEC0xhPYnJEFjQkq2QIMyEIU+NI0AURq6Q4NM/jJZrwjEXIvIc6LItj", + "RKeIcUa2KsRgCxpRTQn9il46GCmREQ9lIoBpTCMPBw6PkXmMjo9QZ06uqosMf5rsB81TMpyQ1Ul/zhLM", + "epq4Giw3P7xbnvv5nm9mypMkG88Ez9LVmY9fnpy8QfAQsSyZEFGecX+Yz0eZIjMi9IRpSMc4igSR0o+/", + "e1iGbTAYDEZ4OBoM+gMflAvCIi4aSWoe+0m6M4jImilbkdTOv0LSF+fHR8cH6JCLlAsMY1dWqgl2mTxl", + "vMpiU+WKT/4PBcHKCn+jKvCj9hJ+4BjNYj7BcbxEGaMfsorc9NGx3gIKpYIvaESiLsLwAFGJcKZ4b0YY", + "EViRCE0FT5CaE1TiLeqQ/qzfRRca3Z5mbg8Pe4NBb3ARVLkT7/VmaRZ0gxQrRYQG8H9/xb3fD3r/HPQe", + "vS1+jvu9t3/5g4+RbQUO8SnAafHsOK50kQO2LIV1QNdL6BomN7PvOMGzW3Pv8BhRPQ4JMiWCMI2JgT/i", + "4Xsi+pRvx3QisFhusxllV6MYKyJVFZv1796IH8C2BjE206jfErXangNx68T8kogQS4JiogVEdlFEZ1TJ", + "LsJabWM5JxJpm/JXFGKmZVYqLBTiAhEWoUuq5gjDe1UKJMseTmmPGlCDbpDgq+eEzbTdfLi7Io9aGDv2", + "R+/tn92ftv6/VyRFFhOPML7imaJshuAxmnKB1JxKVMBAFUlg3B8EmQaj4P9tF87AtvUEth11s5jotRLK", + "js2wnRwSLARe+rnmgFvHPakwW6NXzAby4HfkLJtEVltKpDjC4LcAvs9O32zrLZliKdVc8Gw2L3PlV6cP", + "3pZosULdKpLdIKLy/Zjy8ST1wUTle3S8/RJpbYVimlBVaKedweDk8ba8CPQ/Hrh/bPXRkXFoAHyNPBdW", + "aco5FgRNsCQR4gwdnr5BOI55CMpfOw4hZ1M6ywSJ+jUzBLP7pIWwhYYbRxE1q5xWyO1xBsoIPmELKjhL", + "CFNogQXVm6diXD8GL14ePRk/eXEejDQnoyy0lur05avXwSjYHQwGJboW8jDnKo2z2VjS30nFzQt2nz0O", + "6oAc5PCjhCRcLIHjdg7UmVe395SLBCsU0/cEXej5DBN2ntUV7xCWWiHCfJkSsaCSe5y7n/Nnmn+ZJOW9", + "ZoS7ymJJhPb+HO+AmcA+liVaKsOYZ1GvtGQ3+EASENMCUM9Lq96WVt2ttPoN6hrHKWWkUV93vxUde8nF", + "+5jjqLfzhVUsI0rPvYriC/OgykwrACTnv/Y4Krtsgll0SSM1H0f8kmmQPbrEPkH5y7lCudKY4Pg///r3", + "+UnhUOw8m6RWu+wMH3ymdqnpEz21jzAFIlnqR+NN6kfi/OQ///q3w+TrIkGYls+oonRMtFVF5Zc5UXMi", + "SlbGMVj/yXh7MBw5eSktXwnfyoHoiiLkCyJivPQowp2BRxP+IqiC/WXHIW2hkB58gxrUszljtKoIB35N", + "6AHKA9Njvb+tXm4DSQ7IzvDE/hy21c2LMM1kBaRhHZwXEE1ql3xBhcpwrOWkYra8waVJW3jMvMmKlN0N", + "y/9cHrBCofZvtDZQFDyhVu6WmRlyGKvOh9/DMlq+2cO6IYXji3LzqC3MpOIJohFhik4pEahTC8hoNXSr", + "cmzB416EFQZ93NJoGHBXo99kaaYyTGkSzfFssjrlmZZAytCMzvBkqaoOy85glfV+Qrv5faRuygwZ8SDR", + "WHFPwsNJy/GRpqN7t03WB/JIY8XHiyn1zJxrqiICpRKFtTSUFVo9RS8NqU1LddHlnGrdJpEjAhi085Oy", + "I92/YD2kgRuho3yBfNp8Sm3S9aY37lmHixIQlIG3NFluIYzOT/rodQ7tnyRiWNEFcamyOZZoQghDGdhE", + "EsH6kAAsA5BJHfFQVR9ufXCTVduCeIHbZ32kHbgEM3RJ4xjyDQlWNIRkxYTW8LmcE2YZpVfSCoAVbt4F", + "K0uWTU/WVX43AM1AojFWnqiNzKhUotAcUuEkRZ1XTw93d3cf1ZX08EFvsNPbefB6ZzAa6P//M+gGRrlq", + "Tw4r0rPqZxOJQ99cB1V9YdM/ZY1y+Ob4aGgtQnUd9fsefrR/dYXVo4f0Uj76PZmI2W+7eCOpRb96Oiry", + "VqiTSSJ6TvVpqfJlq0pJoYZs1CcnmW6V1TR/WG9+DHav9Zt3kQet6VVIPsIr3U/IVNaVYGVfNevo15YM", + "VXz0X7V/UEh+KSCzOcOQlqYt6Kpj/seC4PfalffYV22e5djYHX/CINPO62SJyJX2a0mEBOdqKk2QVnVT", + "dvZ+2tvffbi3PxiU9jll6mEp314SYh7ScaitSisAdGQY46UOTvUY1AHvOkKTmE+qwvtg9+H+T4NHO8O2", + "cBjftB0dci/KjUIdS5G/uKMk96QC1HD408Pd3d3Bw4fDvVZQWQevFVDOGay4Dj/t/rS3sz/ca0UFn6//", + "RAiTU6gdKPHII6QHaRpTE9n0ZEpCOqUhInoGpAegTgJmieRudnVPTnA0FtYN9NoDhWnsIUMp1WIWs2+i", + "jrbpSRYrmsbEPAOGtPJ0AfMjmMmXZqOMETEmjjy3mCkhUnrzHrV0hMMlfwVclIhMstlMk6RMuhMqwbMo", + "HCJK4mhkduiNeg64WQD2tkkOLA4tpeE5vySiF5MFictCYMyRBjbhgqBcTgzTKlhRtsAxjcaUpZlXJBpJ", + "+TQT4F+aSRGe8EyBL2kYVl4Ezh0gRphqde0l1go5fiY4NmfKVUpIhZUJ9pxu5u+reTH+/kZ22El8bDh2", + "GbMaAxKPCTw8OTIGPuRMYcqIQAlR2J5gl7LMcNgRdIOelqkIk4QzxKfTv67POzeEAPkGWedEHpZDz7tz", + "IOnMRpR1F1byeEEilGBGp0QqZN8sryznePjg4QhPwp3hbkSmew8e9vt9f3ZGiWXKKfMs9SR/1o4V2ya3", + "2Svm7Mv55/HhDvLpbXD5GJwevP45GAXbmRTbMQ9xvC0nlI1K/87/WTyAH+afE8q8efhc59YgBRVjNYIO", + "V8020lHXFNO4VsyRZnFs/z7SmDAS5gLJQdncGOL6/e8XWjRj+juJkPd0UuGZdsSNxH3eMWQ3+JCRjIxT", + "LqlZfcULtk90KDvJaBwhGFEu7FDmT9XEyLAR/ZJzAumGdZ6JtGkNeMeumTFFYxNx+/01j5/SApRc7dbS", + "fICzfVr4yylhkbGgWgzMr5Czhd4V8A+AT+sZIzgVBe6erTDjkov3lM3GEfVI5y/mIYqoIKGC46Cb91Cw", + "jdP0ZlH0Z4NynZajf0P4YU9SPdblq2vyT4nbq6u/nP39w//I059+2/nw/Pz8H4tnfz96Qf9xHp++/Kzj", + "ovWH6F/1JHxtahaC1coJeFvxOMEq9Dg+cy5VA9XsE6Q4SvTgPjrEDE3I6IL10HOqiMDxCF0EOKV9S8x+", + "yJOLAHXIFQ6VGYU4Q3oqNCc4ImJLDz41R2Z68EcXk13X54iWDCc0RMISOT+Kkdkk4gmmbOuCXTA7F3KI", + "SMj96V8RCnGqMkE0R1CYiXiJJgKHJC/sKRbvoo84Ta+3LpiaY4XIlRIagxQLlVfcuBWA0RYqk1u0r5MI", + "LXCcEYlCINQFy+1HpEHQkygsZkT180gU/P1afq+BKN6EDheqckSxP+h6+Ij0e5qRMZWKMJQfJVIJwos6", + "7oBpf1DZ/vuD/ZvT2LkMrRE/kO7VMk8nlC32hxFgWNoo4/FcqfTmuk3QN2aPoJ9fvz7VZND/PUNuooIW", + "OYs7nMVLhHVcTKRJzqoYfBJ7prcV+BKwhrstEXptXtbDYnkzHk9gYfT6+RlSRCSUGf3dCTU5pzp8JyZN", + "SKXMtChSjA4OT55s9VvUqQJtc/jX8PF1jmEtG+VOpFczYDCiyL1o+nbR8VFXu1N2hxaOFqTfn3KBYqNg", + "in09Qm8kqR6GAatMptBwMl4WVTFGq18EW27GtK4pRuhV7t/hHJS8ErAQBjdlsS9h2gv2ixYMczawMnu3", + "Ciucetj4xao2OAnACtncCZjiZlWwfvt7KA57nrN6wcDt9na50kAv5heNgvd37oHs3jaWvG1VVfVAuVRA", + "kBdWfd2KqNX6JizHkuFUzrlqPrHDyL2DyBWVSq5WE7U6Y1qtpqoaG1MnteaI/kvWRYmMMTguq6PxxSue", + "vuYB1LdXbbW2Pupzi5ysu3VHNU6N29tXH1Td6ebPX7Za6U7AqdQd+ZRB2Sq56oBPLjXqBtRzMnogJZ0x", + "EqHj06KevkhfuOlrOD0a9nce7vd3BoP+zqBNMifB4Zq1Tw4O2y8+GJrwdoQnozAakelnJJOsYBv3AceX", + "eCnRhXPwLgLjUZZcydK2tU5gq3T2akXXpxVw1U3aTSVatynJaqXvoRKzwVE4gyrN23sJDxq9hBu5KhVW", + "5GY33myiM3jZjRrfJs1JUMizOGJ/Umiid55x7Elk4w9JlJEU8y6V6A17z/glq6Jusl16/37IiFii85OT", + "Sm5UkGkm25X/SMXTtJEPPL0VG4Y3OGs3QlOqwNtE1V1dE5Ys0BevsSsnctxhn5G6Fgmdstw1F3/BdJCs", + "MfV60UhLBrKzo0mmUF4ErUXuUPtBqORdmVIniJ9eGUdLzwA2I9RP4mXugK0dfIq1+LmxKfxr/Yizeaa0", + "cYcxcp4ppP8FIGsUrAO7fgojySP0gsMYC2lXq/+aJ2xexyyaLFdfr3vNHZPb0cGc4oJEsJjdliP0NN+K", + "+Wa2m7cjif1pNIQ9FIYD7y0T9Fmn1XIr6AaW6kE3MCQMuoGjjP5pMIRfAHzQDSwg3nqSU6zmx2zKV8O0", + "26gse/jiguJUIymlVjMRYZREW330sqK7LN3gOCeWBEUZseVthg4C24pCbELVFKs5CCYMpGxWLbZeWbCN", + "IjEwrC9nhHXti218Huk/MHgtMqCVCUkkwsXRQav4isrxlMakzcSCzLIYCwTvtwNZLpOYsvdtZpfLZMJj", + "GiI9oG6QpjyO+eVYP5J/A1y2WmGnB4yLLFnNwBjgbI7UMKS2boHC3zSWW7VTl1Bbg20zfhuuXbdxIb1V", + "EE9pTFAC1S9vGL0qCXq1NmhvOGg6ZGuYtHK8Vi37aVVeVLMkVmR9luIVkTwTITnILy54UjRptgrnQhtU", + "d9+hegK758MWsizrjhTzqUrnis4rdbVXVbqWaqBalVw5Y+2tKsxtYsMxk6kW8Abfblq/o39cTkXWo+JF", + "4q990a5yE7VOjCO9Sq9K5u7B/qNHu3sPHg1bkcZGO3m43JAMawqZHQTbkoS1O0JVjg0fDOB/twLKBMx+", + "kBqC5ipAlfs+nwzQ9Zrtc5afUtdq8PP9saYxQ8FJYaersHJvvxW1sOsU4Ql23SPwKErXODtkOiXgqI0N", + "3XoFMLVDnlYwhDjFIVVLzzkovoS8N8pfKc3+sF3JZA1YD0nt3AhPlfb+F0TIbFIU2XXc4ujPCDJJNVnY", + "b11PKrPJGGbwJN3qq8J79qAoqoUgRdDDs0lcSk/bSnFtJkAifHnUy5yY6BLLSmyof4eKRN3SNd16EsG8", + "sa5ce7W2SoOCbNl0KT3qKyWv2SA7qMz+Gju7QdmaFOJcp/g6M9a8BbVVhlOoNmGaxyp66kStXWwzkdUP", + "1g5+2qjxpFzpvbaUvlIWnhuU2y9bSsveZmC9BhXEw8JgKVDM3a1wyMdcEzQ3XXBKXAOhWokqlUrH//bO", + "Dyq9jDokSdXSFUm5mH7rdkH8QT6hVza+8LHW4NGXKKx5s7aS5r/kylw5b+IWuTFjssLTxuNrv/d4VD+T", + "MGGSvTJQzaHXCqGl6jU7l+u6XJl2UxAD2dKRWVavdb1FZ6umqLfYOeaEumhtdVMw13BQbO7TlDArQdLM", + "G5M0+8w2YFS6/l+fSDIbkdxci2FSNDom7NXvlIAXdikohDiWQIawmgR51LoaGq/P5Z/gq3wFCCCxRLWr", + "zwaPUluQZ4/hivsrd7eATt0UAEb9Evvjz+uP5qRqlRnrGqa5tKx341n9s0ajNe2tmnAWa3TX92TTqouE", + "maBqeaYNgj1xJFgQcZAZMQRLAUjAn4vFoR7p+hqixqnHeXxGGBE0RAenxyAlCWZ4pll2foJiOiXhMoyJ", + "LSdZSW1Ce4qXh8c9UwfnTlzh/I8qIIi7b3pwegxX3YQ06w76wz60RuEpYTilwSjY7e/AZT5NBkBxu9QO", + "xyZn9EYEU3YcWZN7ZN/RxJUpZ9K8PxwMzG0SpqxyxcWFou3fpEk7GAPb2kuzF59X8+grhRLOGRBww5Vo", + "SXfIXHeDvcHOrYC78UKQD4Q3DGdqzgX9nUR60Qe3pMgnLXrMTMzr+r4Q+2IhwsHo16rw/vr2+m03kFmS", + "YO0xGtL56ZZy6ZGCcse2wOwxItVjHi2/GL6+pnDX1Q2ttdf1ihB+OT472Vulub2lW5DMiNgGuP0YR3mh", + "WMdesMoL4CpXgb+W0O8N9u5+0dIN8vzqF+Km7M4A8ejugTjkbBrTUKGeg8U2CkM4Nk0JqgJyX9TBKws1", + "wg6vKdRNFl3O9HTOVGxXklKNRqPW5HQz1qPeWfUWZiTHqnRn+4cluUl0jqgMtXNZlpZeiNNSL1dZ7NOy", + "FH2k0bXxlWJizrOrMnQEf89NTooFTogiQgJMDa37UNHnlOoH7iwIwlwTRFbNSbdEw7ov+XZFYvca+zdk", + "rG4bNqAUj2oK8SsqwlpBWamJyX2S5jc5F13ThuuuX8M9I+rbEs3B5rwgd7H/a4r5fZGoZ0S5LZKTTWvB", + "eX4hvUm87JX1O2S0XcGD+JmOPs2uNoCaQqYCLTMUhXMSvjcI2d4i6zyCY9d+5O79AHPv/hbW34L/w9y3", + "CBwLWq0LFo9tddvdxYqVDtStQsXhF4PACpiHyFCaP3EXmU2JHZZLFm59jZjxvzsqrPcCuUc76TSLY+hm", + "Zy+yF90Hyvp0+6P2D1r4yW63rfVF3rx63iMs5BGJ7BWSZofEXTb+st6yYZhB5YeYtImvgFROMJqd0c/g", + "vzk5KBrw/3H41N4c+uPwqbk79Mfdg6IP/90Iy2BTqnnT3us9Fj7tvNIq0UA1mSvBN3l7+Vsbcfhs74Xb", + "uHw5gD+8vjZeX5lcax2/vA3GHbp+1W90bPicIBc2H7Xhkbtt8Z25fJtNPVmJNEekUJlRycXbCwjwFQd7", + "o990CL5PW89WHNBc4sr6t2UOtdiQa70DJ7rHR13brMG0WEgFmdKrzWVUHRwb9xLtuptPpx4kEzrLeCbL", + "t8ihNweRRT/MigK+b/5rYZ4bPdhvWEoHmzQdG3dQf8j9HbnOdYYa5W2ORW5ynt1bm3Gei6Oa9t6zg/CH", + "99zKey6Ra733nF8+vkv3ufqRtI37z07efAS3dZXfowd9z7xSzGyOu3TYW9FxrR3UouXLettffN9k4wf9", + "+eKb90vdXcP7mEOC7hHwITbnCRa2ptkV/NbkYbBZ3bd5F/A+i9izcptBv7MFimg75rOy21VvcyIITop2", + "Yki/jbBEZwBY74wwhZ4sNFb9C+Z6Er4zl6TeoVxQzRcZYxIq+92jmMN3fSTMD3053uE0fZf33twaoWdQ", + "3lmirlm8I4mgOEYhZ5LHpr/Fu0WSvBut1omfn5zAIHhnbirC343ybxHle0zqty7YBXtFVCaYBCxiLBV6", + "gWLKiEQdzXDB49h8p+KdpmcJvy1ocKhnNM3t4uUF0yMoy4i0WFI2Q4xc2gnpFL0zjQPg/sE70+uwcdc/", + "11z6Sju/29xzxuCiOBJAONMvksAHCGBd6MFTLGw/jlAsld+i2Bl4rzt9XE10AU29JDV3YSmjSssHz5T5", + "4IIPEEN5PyiNV3xWvw0xQ+6iaEWUcZq2FV8LJkjxIknWyDDqFF35kFQRz9RfpIqIMG2ErXQ3CTfq4ND8", + "Q+H3pultpU+g6briI5W9leslVWBae7tmLeZfiyQJTNPCBPuar7SwJIpcqW2i1UrPkLWqU+sTrsZjmjMw", + "EHXOzp5s/bAZLd0SIFlV2VsCeiyH7foDN9W8wdsr88J377m49khfWQw3fxRRgoJC4zYWTZb2w4B5l9Z7", + "dScAGFlgBvbO4uXdI+5Z4x6x7aq++z1SyMd3vktCLqAPu3SdFO9P8VYp4iht9w40uSuax3Vd1Ht+crLV", + "tGlMK+/GLSN+hMO2jvK7tynQ9+/+7RbTyBXnCKxLFuoNoRpjdBezUmbaZehQw3wrDq+2TID+eHIpFUlM", + "wD7NYrjYBlXr8F3fqRtnagW68KVfLf5dSFmVer9dsAmZanuYEqHX1sP1/KXYwxfWnimcb99Tswe/jbgW", + "uihAKIdVE9VWPm3kGij4Yqe858Mng/QUAtVq/0GJOvBJdwBzIVGsf2ytjXRNc8Lbxbt3qeHy9pu+W61G", + "ZnNh/h403HFNrblWsvdOrT0j5c3i9A8w2qfWeLrOzPP0h5W3fW1/+MT30ieGg54cm85M4BAsrrSdi/3+", + "r20Buv3R/Di+6bhQ4XB+7tpPfRum1HaruWkZh+C92JQWp4iYK72b35M8byh0T69taMI5FCB1Uj749FsB", + "06jse5PuL1/jUqbjrSpcNrq33HX5b2ZvbdryWRhcuXaZHvdlmxtJc5goXgttRbmB6dqA1jW0hG66blje", + "CbZbbu9rPg+bB6hFI7q8k2j/guWtUxFlYZxFBB2evunaz7V04YMwZgbbsLOP/B1uJcKCuDa3F0xxFOI4", + "zGKsCMpbvZr2zLLhWPdVqf3xne23YhEPo/MetzLvgXqfYgy/TAD3yk1WQeJK3zpprC21nz3ZSGWpNWa3", + "qCt1GPwowWtRVVoilnMpfA0rJcJQO2Be76OzLE25UBKpSw7fOpBwlv/3s5cv0IRHyxHKxzFk2upaFWf7", + "ocqUhHRKSQQ9KfXYE6jWxgI+qpqUJnAjU0F6KU9BdUTmyo6lsXGPMFJY9Ge/IyzCOV0QjzIxc+b+0d2V", + "x9Zdh26QOPS2NXrQBbc6ae3jCgUsVX5UcTQZRPtNcfM9w7ypqZui1Bl4QhkWy7ZtgV/a+hwUZlLxxM17", + "fIQ6uPoFQ9syORV8QaN6S/FvpH/wCb6iSZbkH9F49hg+SSBMqQd8nAYKjZxMkauQkEhC5cfWLXsNr7YZ", + "trzw9A3daN2006aNPuVXrJkumhNqFmsf0wm54hzFWMzI1ndzM9HuteJi4vFR7VriPaz2XjjpK/yMlvXd", + "7ULalpHmXdR25+mOzVZ2n387UVipf9s9vF64yN3MppLyb0sEB5szCZsuJT+/x1k7HW0tamQzE+gZfQLz", + "nIc4RhFZkJin0E3fvBt0g0zEtjf4aHtbh2mxDuTg6+vB9dvr/wsAAP//kZMzgCqrAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 82c9606..49a3518 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -18,6 +18,7 @@ import ( hypemanotel "github.com/onkernel/hypeman/lib/otel" "github.com/onkernel/hypeman/lib/paths" "github.com/onkernel/hypeman/lib/registry" + "github.com/onkernel/hypeman/lib/resources" "github.com/onkernel/hypeman/lib/system" "github.com/onkernel/hypeman/lib/volumes" "go.opentelemetry.io/otel" @@ -45,9 +46,14 @@ func ProvideContext(log *slog.Logger) context.Context { return logger.AddToContext(context.Background(), log) } -// ProvideConfig provides the application configuration +// ProvideConfig provides the application configuration. +// Panics if configuration is invalid (prevents startup with bad config). func ProvideConfig() *config.Config { - return config.Load() + cfg := config.Load() + if err := cfg.Validate(); err != nil { + panic(fmt.Sprintf("invalid configuration: %v", err)) + } + return cfg } // ProvidePaths provides the paths abstraction @@ -140,6 +146,23 @@ func ProvideRegistry(p *paths.Paths, imageManager images.Manager) (*registry.Reg return registry.New(p, imageManager) } +// ProvideResourceManager provides the resource manager for capacity tracking +func ProvideResourceManager(ctx context.Context, cfg *config.Config, p *paths.Paths, imageManager images.Manager, instanceManager instances.Manager, volumeManager volumes.Manager) (*resources.Manager, error) { + mgr := resources.NewManager(cfg, p) + + // Managers implement the lister interfaces directly + mgr.SetImageLister(imageManager) + mgr.SetInstanceLister(instanceManager) + mgr.SetVolumeLister(volumeManager) + + // Initialize resource discovery + if err := mgr.Initialize(ctx); err != nil { + return nil, fmt.Errorf("initialize resource manager: %w", err) + } + + return mgr, nil +} + // ProvideIngressManager provides the ingress manager func ProvideIngressManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager) (ingress.Manager, error) { // Parse DNS provider - fail if invalid diff --git a/lib/resources/README.md b/lib/resources/README.md new file mode 100644 index 0000000..22d2935 --- /dev/null +++ b/lib/resources/README.md @@ -0,0 +1,132 @@ +# Resource Management + +Host resource discovery, capacity tracking, and oversubscription-aware allocation management for CPU, memory, disk, and network. + +## Features + +- **Resource Discovery**: Automatically detects host capacity from `/proc/cpuinfo`, `/proc/meminfo`, filesystem stats, and network interface speed +- **Oversubscription**: Configurable ratios per resource type (e.g., 2x CPU oversubscription) +- **Allocation Tracking**: Tracks resource usage across all running instances +- **Bidirectional Network Rate Limiting**: Separate download/upload limits with fair sharing +- **API Endpoint**: `GET /resources` returns capacity, allocations, and per-instance breakdown + +## Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `OVERSUB_CPU` | `4.0` | CPU oversubscription ratio | +| `OVERSUB_MEMORY` | `1.0` | Memory oversubscription ratio | +| `OVERSUB_DISK` | `1.0` | Disk oversubscription ratio | +| `OVERSUB_NETWORK` | `2.0` | Network oversubscription ratio | +| `OVERSUB_DISK_IO` | `2.0` | Disk I/O oversubscription ratio | +| `DISK_LIMIT` | auto | Hard disk limit (e.g., `500GB`), auto-detects from filesystem | +| `NETWORK_LIMIT` | auto | Hard network limit (e.g., `10Gbps`), auto-detects from uplink speed | +| `DISK_IO_LIMIT` | `1GB/s` | Hard disk I/O limit (e.g., `500MB/s`, `2GB/s`) | +| `MAX_IMAGE_STORAGE` | `0.2` | Max image storage as fraction of disk (OCI cache + rootfs) | +| `UPLOAD_BURST_MULTIPLIER` | `4` | Multiplier for upload burst ceiling (HTB ceil = rate × multiplier) | +| `DOWNLOAD_BURST_MULTIPLIER` | `4` | Multiplier for download burst bucket (TBF bucket size) | + +## Resource Types + +### CPU +- Discovered from `/proc/cpuinfo` (threads × cores × sockets) +- Allocated = sum of `vcpus` from active instances + +### Memory +- Discovered from `/proc/meminfo` (`MemTotal`) +- Allocated = sum of `size + hotplug_size` from active instances + +### Disk +- Discovered via `statfs()` on DataDir, or configured via `DISK_LIMIT` +- Allocated = images (rootfs) + OCI cache + volumes + overlays (rootfs + volume) +- Image pulls blocked when <5GB available or image storage exceeds `MAX_IMAGE_STORAGE` + +### Network + +Bidirectional rate limiting with separate download and upload controls: + +**Downloads (external → VM):** +- TBF (Token Bucket Filter) shaping on each TAP device egress +- Simple per-VM caps, independent of other VMs +- Smooth traffic shaping (queues packets, doesn't drop) + +**Uploads (VM → external):** +- HTB (Hierarchical Token Bucket) on bridge egress +- Per-VM classes with guaranteed rate and burst ceiling +- Fair sharing when VMs contend for bandwidth +- fq_codel leaf qdisc for low latency under load + +**Default limits:** +- Proportional to CPU: `(vcpus / cpu_capacity) * network_capacity` +- Symmetric download/upload by default +- Upload ceiling = 4x guaranteed rate by default (configurable via `UPLOAD_BURST_MULTIPLIER`) +- Download burst bucket = 4x rate by default (configurable via `DOWNLOAD_BURST_MULTIPLIER`) + +**Capacity tracking:** +- Uses max(download, upload) per instance since they share physical link + +### Disk I/O + +Per-VM disk I/O rate limiting with burst support: + +- **Cloud Hypervisor**: Uses native `RateLimiterConfig` with token bucket +- **QEMU**: Uses drive `throttling.bps-total` options +- **Default**: Proportional to CPU: `(vcpus / cpu_capacity) * disk_io_capacity * 2.0` +- **Burst**: 4x sustained rate (allows fast cold starts) + +## Example: Default Limits + +**Host**: 16-core server with 10Gbps NIC (default disk I/O = 1GB/s) + +**VM**: 2 vCPUs (12.5% of host) + +| Resource | Calculation | Default Limit | +|----------|-------------|---------------| +| Network (down/up) | 10Gbps × 2.0 × 12.5% | 2.5 Gbps (312 MB/s) | +| Disk I/O (sustained) | 1GB/s × 2.0 × 12.5% | 250 MB/s | +| Disk I/O (burst) | 250 MB/s × 4 | 1 GB/s | + +## Effective Limits + +The effective allocatable capacity is: + +``` +effective_limit = capacity × oversub_ratio +available = effective_limit - allocated +``` + +For example, with 64 CPUs and `OVERSUB_CPU=2.0`, up to 128 vCPUs can be allocated across instances. + +## API Response + +```json +{ + "cpu": { + "capacity": 64, + "effective_limit": 128, + "allocated": 48, + "available": 80, + "oversub_ratio": 2.0 + }, + "memory": { ... }, + "disk": { ... }, + "network": { ... }, + "disk_breakdown": { + "images_bytes": 214748364800, + "oci_cache_bytes": 53687091200, + "volumes_bytes": 107374182400, + "overlays_bytes": 227633306624 + }, + "allocations": [ + { + "instance_id": "abc123", + "instance_name": "my-vm", + "cpu": 4, + "memory_bytes": 8589934592, + "disk_bytes": 10737418240, + "network_download_bps": 125000000, + "network_upload_bps": 125000000 + } + ] +} +``` diff --git a/lib/resources/cpu.go b/lib/resources/cpu.go new file mode 100644 index 0000000..883cbff --- /dev/null +++ b/lib/resources/cpu.go @@ -0,0 +1,142 @@ +package resources + +import ( + "bufio" + "context" + "fmt" + "os" + "strconv" + "strings" +) + +// CPUResource implements Resource for CPU discovery and tracking. +type CPUResource struct { + capacity int64 + instanceLister InstanceLister +} + +// NewCPUResource discovers host CPU capacity from /proc/cpuinfo. +func NewCPUResource() (*CPUResource, error) { + capacity, err := detectCPUCapacity() + if err != nil { + return nil, err + } + return &CPUResource{capacity: capacity}, nil +} + +// SetInstanceLister sets the instance lister for allocation calculations. +func (c *CPUResource) SetInstanceLister(lister InstanceLister) { + c.instanceLister = lister +} + +// Type returns the resource type. +func (c *CPUResource) Type() ResourceType { + return ResourceCPU +} + +// Capacity returns the total number of vCPUs available on the host. +func (c *CPUResource) Capacity() int64 { + return c.capacity +} + +// Allocated returns the total vCPUs allocated to running instances. +func (c *CPUResource) Allocated(ctx context.Context) (int64, error) { + if c.instanceLister == nil { + return 0, nil + } + + instances, err := c.instanceLister.ListInstanceAllocations(ctx) + if err != nil { + return 0, err + } + + var total int64 + for _, inst := range instances { + if isActiveState(inst.State) { + total += int64(inst.Vcpus) + } + } + return total, nil +} + +// detectCPUCapacity reads /proc/cpuinfo to determine total vCPU count. +// Returns threads × cores × sockets. +func detectCPUCapacity() (int64, error) { + file, err := os.Open("/proc/cpuinfo") + if err != nil { + return 0, fmt.Errorf("open /proc/cpuinfo: %w", err) + } + defer file.Close() + + var ( + siblings int + physicalIDs = make(map[int]bool) + hasSiblings bool + hasPhysicalID bool + ) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch key { + case "siblings": + if !hasSiblings { + siblings, _ = strconv.Atoi(value) + hasSiblings = true + } + case "physical id": + physicalID, _ := strconv.Atoi(value) + physicalIDs[physicalID] = true + hasPhysicalID = true + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + // Calculate total vCPUs + if hasSiblings && hasPhysicalID { + // siblings = threads per socket, physicalIDs = number of sockets + sockets := len(physicalIDs) + if sockets < 1 { + sockets = 1 + } + return int64(siblings * sockets), nil + } + + // Fallback: count processor entries + file.Seek(0, 0) + scanner = bufio.NewScanner(file) + count := 0 + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "processor") { + count++ + } + } + if count > 0 { + return int64(count), nil + } + + // Ultimate fallback + return 1, nil +} + +// isActiveState returns true if the instance state indicates it's consuming resources. +func isActiveState(state string) bool { + switch state { + case "Running", "Paused", "Created": + return true + default: + return false + } +} diff --git a/lib/resources/disk.go b/lib/resources/disk.go new file mode 100644 index 0000000..483cf3f --- /dev/null +++ b/lib/resources/disk.go @@ -0,0 +1,128 @@ +package resources + +import ( + "context" + "strings" + "syscall" + + "github.com/c2h5oh/datasize" + "github.com/onkernel/hypeman/cmd/api/config" + "github.com/onkernel/hypeman/lib/paths" +) + +// DiskResource implements Resource for disk space discovery and tracking. +type DiskResource struct { + capacity int64 // bytes + dataDir string + instanceLister InstanceLister + imageLister ImageLister + volumeLister VolumeLister +} + +// NewDiskResource discovers disk capacity for the data directory. +// If cfg.DiskLimit is set, uses that as capacity; otherwise auto-detects via statfs. +func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { + var capacity int64 + + if cfg.DiskLimit != "" { + // Parse configured limit + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { + return nil, err + } + capacity = int64(ds.Bytes()) + } else { + // Auto-detect from filesystem + var stat syscall.Statfs_t + if err := syscall.Statfs(cfg.DataDir, &stat); err != nil { + return nil, err + } + // Total space = blocks * block size + capacity = int64(stat.Blocks) * int64(stat.Bsize) + } + + return &DiskResource{ + capacity: capacity, + dataDir: cfg.DataDir, + instanceLister: instLister, + imageLister: imgLister, + volumeLister: volLister, + }, nil +} + +// Type returns the resource type. +func (d *DiskResource) Type() ResourceType { + return ResourceDisk +} + +// Capacity returns the total disk space in bytes. +func (d *DiskResource) Capacity() int64 { + return d.capacity +} + +// Allocated returns total disk space used by images, OCI cache, volumes, and overlays. +func (d *DiskResource) Allocated(ctx context.Context) (int64, error) { + breakdown, err := d.GetBreakdown(ctx) + if err != nil { + return 0, err + } + return breakdown.Images + breakdown.OCICache + breakdown.Volumes + breakdown.Overlays, nil +} + +// GetBreakdown returns disk usage broken down by category. +func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) { + var breakdown DiskBreakdown + + // Get image sizes (exported rootfs disks) + if d.imageLister != nil { + imageBytes, err := d.imageLister.TotalImageBytes(ctx) + if err == nil { + breakdown.Images = imageBytes + } + // Get OCI layer cache size + ociCacheBytes, err := d.imageLister.TotalOCICacheBytes(ctx) + if err == nil { + breakdown.OCICache = ociCacheBytes + } + } + + // Get volume sizes + if d.volumeLister != nil { + volumeBytes, err := d.volumeLister.TotalVolumeBytes(ctx) + if err == nil { + breakdown.Volumes = volumeBytes + } + } + + // Get overlay sizes from instances (rootfs overlays + volume overlays) + if d.instanceLister != nil { + instances, err := d.instanceLister.ListInstanceAllocations(ctx) + if err == nil { + for _, inst := range instances { + if isActiveState(inst.State) { + breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes + } + } + } + } + + return &breakdown, nil +} + +// parseDiskIOLimit parses a disk I/O limit string like "500MB/s", "1GB/s". +// Returns bytes per second. +func parseDiskIOLimit(limit string) (int64, error) { + limit = strings.TrimSpace(limit) + limit = strings.ToLower(limit) + + // Remove "/s" or "ps" suffix if present + limit = strings.TrimSuffix(limit, "/s") + limit = strings.TrimSuffix(limit, "ps") + + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(limit)); err != nil { + return 0, err + } + + return int64(ds.Bytes()), nil +} diff --git a/lib/resources/memory.go b/lib/resources/memory.go new file mode 100644 index 0000000..52cebd7 --- /dev/null +++ b/lib/resources/memory.go @@ -0,0 +1,91 @@ +package resources + +import ( + "bufio" + "context" + "fmt" + "os" + "strconv" + "strings" +) + +// MemoryResource implements Resource for memory discovery and tracking. +type MemoryResource struct { + capacity int64 // bytes + instanceLister InstanceLister +} + +// NewMemoryResource discovers host memory capacity from /proc/meminfo. +func NewMemoryResource() (*MemoryResource, error) { + capacity, err := detectMemoryCapacity() + if err != nil { + return nil, err + } + return &MemoryResource{capacity: capacity}, nil +} + +// SetInstanceLister sets the instance lister for allocation calculations. +func (m *MemoryResource) SetInstanceLister(lister InstanceLister) { + m.instanceLister = lister +} + +// Type returns the resource type. +func (m *MemoryResource) Type() ResourceType { + return ResourceMemory +} + +// Capacity returns the total memory in bytes available on the host. +func (m *MemoryResource) Capacity() int64 { + return m.capacity +} + +// Allocated returns the total memory allocated to running instances. +func (m *MemoryResource) Allocated(ctx context.Context) (int64, error) { + if m.instanceLister == nil { + return 0, nil + } + + instances, err := m.instanceLister.ListInstanceAllocations(ctx) + if err != nil { + return 0, err + } + + var total int64 + for _, inst := range instances { + if isActiveState(inst.State) { + total += inst.MemoryBytes + } + } + return total, nil +} + +// detectMemoryCapacity reads /proc/meminfo to determine total memory. +func detectMemoryCapacity() (int64, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "MemTotal:") { + // Format: "MemTotal: 16384000 kB" + fields := strings.Fields(line) + if len(fields) >= 2 { + kb, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse MemTotal: %w", err) + } + return kb * 1024, nil // Convert KB to bytes + } + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + return 0, fmt.Errorf("MemTotal not found in /proc/meminfo") +} diff --git a/lib/resources/network.go b/lib/resources/network.go new file mode 100644 index 0000000..1de9a75 --- /dev/null +++ b/lib/resources/network.go @@ -0,0 +1,184 @@ +package resources + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + + "github.com/c2h5oh/datasize" + "github.com/onkernel/hypeman/cmd/api/config" + "github.com/vishvananda/netlink" +) + +// NetworkResource implements Resource for network bandwidth discovery and tracking. +type NetworkResource struct { + capacity int64 // bytes per second + instanceLister InstanceLister +} + +// NewNetworkResource discovers network capacity. +// If cfg.NetworkLimit is set, uses that; otherwise auto-detects from uplink interface. +func NewNetworkResource(cfg *config.Config, instLister InstanceLister) (*NetworkResource, error) { + var capacity int64 + + if cfg.NetworkLimit != "" { + // Parse configured limit (e.g., "10Gbps", "1GB/s") + parsed, err := ParseBandwidth(cfg.NetworkLimit) + if err != nil { + return nil, fmt.Errorf("parse network limit: %w", err) + } + capacity = parsed + } else { + // Auto-detect from uplink interface + uplink, err := getUplinkInterface(cfg.UplinkInterface) + if err != nil { + // No uplink found - network limiting disabled + capacity = 0 + } else { + speed, err := getInterfaceSpeed(uplink) + if err != nil || speed <= 0 { + // Speed detection failed - network limiting disabled + capacity = 0 + } else { + // speed is in Mbps, convert to bytes/sec + capacity = speed * 1000 * 1000 / 8 + } + } + } + + return &NetworkResource{ + capacity: capacity, + instanceLister: instLister, + }, nil +} + +// Type returns the resource type. +func (n *NetworkResource) Type() ResourceType { + return ResourceNetwork +} + +// Capacity returns the network capacity in bytes per second. +func (n *NetworkResource) Capacity() int64 { + return n.capacity +} + +// Allocated returns total network bandwidth allocated to running instances. +// Uses the max of download/upload per instance since they share the physical link. +func (n *NetworkResource) Allocated(ctx context.Context) (int64, error) { + if n.instanceLister == nil { + return 0, nil + } + + instances, err := n.instanceLister.ListInstanceAllocations(ctx) + if err != nil { + return 0, err + } + + var total int64 + for _, inst := range instances { + if isActiveState(inst.State) { + // Use max of download/upload since they share the same physical link + // This is conservative - actual usage depends on traffic direction + alloc := inst.NetworkDownloadBps + if inst.NetworkUploadBps > alloc { + alloc = inst.NetworkUploadBps + } + total += alloc + } + } + return total, nil +} + +// getUplinkInterface returns the uplink interface name. +// Uses explicit config if set, otherwise auto-detects from default route. +func getUplinkInterface(configured string) (string, error) { + if configured != "" { + return configured, nil + } + + // Auto-detect from default route + routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) + if err != nil { + return "", fmt.Errorf("list routes: %w", err) + } + + for _, route := range routes { + // Default route has no destination (Dst is nil or 0.0.0.0/0) + if route.Dst == nil || route.Dst.IP.IsUnspecified() { + if route.LinkIndex > 0 { + link, err := netlink.LinkByIndex(route.LinkIndex) + if err == nil { + return link.Attrs().Name, nil + } + } + } + } + + return "", fmt.Errorf("no default route found") +} + +// getInterfaceSpeed reads the link speed from /sys/class/net/{iface}/speed. +// Returns speed in Mbps, or -1 for virtual interfaces. +func getInterfaceSpeed(iface string) (int64, error) { + path := fmt.Sprintf("/sys/class/net/%s/speed", iface) + data, err := os.ReadFile(path) + if err != nil { + return -1, err + } + + speed, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return -1, err + } + + return speed, nil +} + +// ParseBandwidth parses a bandwidth string like "10Gbps", "1GB/s", "125MB/s". +// Handles both bit-based (bps) and byte-based (/s) formats. +// Returns bytes per second. +func ParseBandwidth(limit string) (int64, error) { + limit = strings.TrimSpace(limit) + limit = strings.ToLower(limit) + + // Handle bps variants (bits per second) + if strings.HasSuffix(limit, "bps") { + // Remove "bps" suffix + numPart := strings.TrimSuffix(limit, "bps") + numPart = strings.TrimSpace(numPart) + + // Check for multiplier prefix + var multiplier int64 = 1 + if strings.HasSuffix(numPart, "g") { + multiplier = 1000 * 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "g") + } else if strings.HasSuffix(numPart, "m") { + multiplier = 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "m") + } else if strings.HasSuffix(numPart, "k") { + multiplier = 1000 + numPart = strings.TrimSuffix(numPart, "k") + } + + bits, err := strconv.ParseInt(strings.TrimSpace(numPart), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid number: %s", numPart) + } + + // Convert bits to bytes + return (bits * multiplier) / 8, nil + } + + // Handle byte-based variants (e.g., "125MB/s", "1GB") + limit = strings.TrimSuffix(limit, "/s") + limit = strings.TrimSuffix(limit, "ps") + + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(limit)); err != nil { + return 0, fmt.Errorf("parse as bytes: %w", err) + } + + return int64(ds.Bytes()), nil +} diff --git a/lib/resources/resource.go b/lib/resources/resource.go new file mode 100644 index 0000000..dce2083 --- /dev/null +++ b/lib/resources/resource.go @@ -0,0 +1,498 @@ +// Package resources provides host resource discovery, capacity tracking, +// and oversubscription-aware allocation management for CPU, memory, disk, and network. +package resources + +import ( + "context" + "fmt" + "sync" + + "github.com/onkernel/hypeman/cmd/api/config" + "github.com/onkernel/hypeman/lib/logger" + "github.com/onkernel/hypeman/lib/paths" +) + +// ResourceType identifies a type of host resource. +type ResourceType string + +const ( + ResourceCPU ResourceType = "cpu" + ResourceMemory ResourceType = "memory" + ResourceDisk ResourceType = "disk" + ResourceNetwork ResourceType = "network" +) + +// Resource represents a discoverable and allocatable host resource. +type Resource interface { + // Type returns the resource type identifier. + Type() ResourceType + + // Capacity returns the raw host capacity (before oversubscription). + Capacity() int64 + + // Allocated returns current total allocation across all instances. + Allocated(ctx context.Context) (int64, error) +} + +// ResourceStatus represents the current state of a resource type. +type ResourceStatus struct { + Type ResourceType `json:"type"` + Capacity int64 `json:"capacity"` // Raw host capacity + EffectiveLimit int64 `json:"effective_limit"` // Capacity * oversubscription ratio + Allocated int64 `json:"allocated"` // Currently allocated + Available int64 `json:"available"` // EffectiveLimit - Allocated + OversubRatio float64 `json:"oversub_ratio"` // Oversubscription ratio applied + Source string `json:"source,omitempty"` // How capacity was determined (e.g., "detected", "configured") +} + +// AllocationBreakdown shows per-instance resource allocations. +type AllocationBreakdown struct { + InstanceID string `json:"instance_id"` + InstanceName string `json:"instance_name"` + CPU int `json:"cpu"` + MemoryBytes int64 `json:"memory_bytes"` + DiskBytes int64 `json:"disk_bytes"` + NetworkDownloadBps int64 `json:"network_download_bps"` // External→VM + NetworkUploadBps int64 `json:"network_upload_bps"` // VM→External +} + +// DiskBreakdown shows disk usage by category. +type DiskBreakdown struct { + Images int64 `json:"images_bytes"` // Exported rootfs disk files + OCICache int64 `json:"oci_cache_bytes"` // OCI layer cache (shared blobs) + Volumes int64 `json:"volumes_bytes"` + Overlays int64 `json:"overlays_bytes"` // Rootfs overlays + volume overlays +} + +// FullResourceStatus is the complete resource status for the API response. +type FullResourceStatus struct { + CPU ResourceStatus `json:"cpu"` + Memory ResourceStatus `json:"memory"` + Disk ResourceStatus `json:"disk"` + Network ResourceStatus `json:"network"` + DiskDetail *DiskBreakdown `json:"disk_breakdown,omitempty"` + Allocations []AllocationBreakdown `json:"allocations"` +} + +// InstanceLister provides access to instance data for allocation calculations. +type InstanceLister interface { + // ListInstanceAllocations returns resource allocations for all instances. + ListInstanceAllocations(ctx context.Context) ([]InstanceAllocation, error) +} + +// InstanceAllocation represents the resources allocated to a single instance. +type InstanceAllocation struct { + ID string + Name string + Vcpus int + MemoryBytes int64 // Size + HotplugSize + OverlayBytes int64 // Rootfs overlay size + VolumeOverlayBytes int64 // Sum of volume overlay sizes + NetworkDownloadBps int64 // Download rate limit (external→VM) + NetworkUploadBps int64 // Upload rate limit (VM→external) + State string // Only count running/paused/created instances + VolumeBytes int64 // Sum of attached volume base sizes (for per-instance reporting) +} + +// ImageLister provides access to image sizes for disk calculations. +type ImageLister interface { + // TotalImageBytes returns the total size of all images on disk. + TotalImageBytes(ctx context.Context) (int64, error) + // TotalOCICacheBytes returns the total size of the OCI layer cache. + TotalOCICacheBytes(ctx context.Context) (int64, error) +} + +// VolumeLister provides access to volume sizes for disk calculations. +type VolumeLister interface { + // TotalVolumeBytes returns the total size of all volumes. + TotalVolumeBytes(ctx context.Context) (int64, error) +} + +// Manager coordinates resource discovery and allocation tracking. +type Manager struct { + cfg *config.Config + paths *paths.Paths + + mu sync.RWMutex + resources map[ResourceType]Resource + + // Dependencies for allocation calculations + instanceLister InstanceLister + imageLister ImageLister + volumeLister VolumeLister +} + +// NewManager creates a new resource manager. +func NewManager(cfg *config.Config, p *paths.Paths) *Manager { + return &Manager{ + cfg: cfg, + paths: p, + resources: make(map[ResourceType]Resource), + } +} + +// SetInstanceLister sets the instance lister for allocation calculations. +func (m *Manager) SetInstanceLister(lister InstanceLister) { + m.mu.Lock() + defer m.mu.Unlock() + m.instanceLister = lister +} + +// SetImageLister sets the image lister for disk calculations. +func (m *Manager) SetImageLister(lister ImageLister) { + m.mu.Lock() + defer m.mu.Unlock() + m.imageLister = lister +} + +// SetVolumeLister sets the volume lister for disk calculations. +func (m *Manager) SetVolumeLister(lister VolumeLister) { + m.mu.Lock() + defer m.mu.Unlock() + m.volumeLister = lister +} + +// Initialize discovers host resources and registers them. +// Must be called after setting listers and before using the manager. +func (m *Manager) Initialize(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + // Discover CPU + cpu, err := NewCPUResource() + if err != nil { + return fmt.Errorf("discover CPU: %w", err) + } + cpu.SetInstanceLister(m.instanceLister) + m.resources[ResourceCPU] = cpu + + // Discover memory + mem, err := NewMemoryResource() + if err != nil { + return fmt.Errorf("discover memory: %w", err) + } + mem.SetInstanceLister(m.instanceLister) + m.resources[ResourceMemory] = mem + + // Discover disk + disk, err := NewDiskResource(m.cfg, m.paths, m.instanceLister, m.imageLister, m.volumeLister) + if err != nil { + return fmt.Errorf("discover disk: %w", err) + } + m.resources[ResourceDisk] = disk + + // Discover network + net, err := NewNetworkResource(m.cfg, m.instanceLister) + if err != nil { + return fmt.Errorf("discover network: %w", err) + } + m.resources[ResourceNetwork] = net + + return nil +} + +// GetOversubRatio returns the oversubscription ratio for a resource type. +func (m *Manager) GetOversubRatio(rt ResourceType) float64 { + switch rt { + case ResourceCPU: + return m.cfg.OversubCPU + case ResourceMemory: + return m.cfg.OversubMemory + case ResourceDisk: + return m.cfg.OversubDisk + case ResourceNetwork: + return m.cfg.OversubNetwork + default: + return 1.0 + } +} + +// GetStatus returns the current status of a specific resource type. +func (m *Manager) GetStatus(ctx context.Context, rt ResourceType) (*ResourceStatus, error) { + m.mu.RLock() + res, ok := m.resources[rt] + m.mu.RUnlock() + + if !ok { + return nil, fmt.Errorf("unknown resource type: %s", rt) + } + + capacity := res.Capacity() + ratio := m.GetOversubRatio(rt) + effectiveLimit := int64(float64(capacity) * ratio) + + allocated, err := res.Allocated(ctx) + if err != nil { + return nil, fmt.Errorf("get allocated %s: %w", rt, err) + } + + available := effectiveLimit - allocated + if available < 0 { + available = 0 + } + + status := &ResourceStatus{ + Type: rt, + Capacity: capacity, + EffectiveLimit: effectiveLimit, + Allocated: allocated, + Available: available, + OversubRatio: ratio, + } + + // Add source info for network + if rt == ResourceNetwork { + if m.cfg.NetworkLimit != "" { + status.Source = "configured" + } else { + status.Source = "detected" + } + } + + return status, nil +} + +// GetFullStatus returns the complete resource status for all resource types. +func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error) { + cpuStatus, err := m.GetStatus(ctx, ResourceCPU) + if err != nil { + return nil, err + } + + memStatus, err := m.GetStatus(ctx, ResourceMemory) + if err != nil { + return nil, err + } + + diskStatus, err := m.GetStatus(ctx, ResourceDisk) + if err != nil { + return nil, err + } + + netStatus, err := m.GetStatus(ctx, ResourceNetwork) + if err != nil { + return nil, err + } + + // Get disk breakdown + var diskBreakdown *DiskBreakdown + m.mu.RLock() + diskRes := m.resources[ResourceDisk] + m.mu.RUnlock() + if disk, ok := diskRes.(*DiskResource); ok { + breakdown, err := disk.GetBreakdown(ctx) + if err == nil { + diskBreakdown = breakdown + } + } + + // Get per-instance allocations + var allocations []AllocationBreakdown + m.mu.RLock() + lister := m.instanceLister + m.mu.RUnlock() + + if lister != nil { + instances, err := lister.ListInstanceAllocations(ctx) + if err != nil { + log := logger.FromContext(ctx) + log.WarnContext(ctx, "failed to list instance allocations for resource status", "error", err) + } else { + for _, inst := range instances { + // Only include active instances + if inst.State == "Running" || inst.State == "Paused" || inst.State == "Created" { + allocations = append(allocations, AllocationBreakdown{ + InstanceID: inst.ID, + InstanceName: inst.Name, + CPU: inst.Vcpus, + MemoryBytes: inst.MemoryBytes, + DiskBytes: inst.OverlayBytes + inst.VolumeBytes, + NetworkDownloadBps: inst.NetworkDownloadBps, + NetworkUploadBps: inst.NetworkUploadBps, + }) + } + } + } + } + + return &FullResourceStatus{ + CPU: *cpuStatus, + Memory: *memStatus, + Disk: *diskStatus, + Network: *netStatus, + DiskDetail: diskBreakdown, + Allocations: allocations, + }, nil +} + +// CanAllocate checks if the requested amount can be allocated for a resource type. +func (m *Manager) CanAllocate(ctx context.Context, rt ResourceType, amount int64) (bool, error) { + status, err := m.GetStatus(ctx, rt) + if err != nil { + return false, err + } + return amount <= status.Available, nil +} + +// CPUCapacity returns the raw CPU capacity (number of vCPUs). +func (m *Manager) CPUCapacity() int64 { + m.mu.RLock() + defer m.mu.RUnlock() + if cpu, ok := m.resources[ResourceCPU]; ok { + return cpu.Capacity() + } + return 0 +} + +// NetworkCapacity returns the raw network capacity in bytes/sec. +func (m *Manager) NetworkCapacity() int64 { + m.mu.RLock() + defer m.mu.RUnlock() + if net, ok := m.resources[ResourceNetwork]; ok { + return net.Capacity() + } + return 0 +} + +// DiskIOCapacity returns the disk I/O capacity in bytes/sec. +// Uses configured DISK_IO_LIMIT if set, otherwise defaults to 1 GB/s. +func (m *Manager) DiskIOCapacity() int64 { + if m.cfg.DiskIOLimit == "" { + return 1 * 1000 * 1000 * 1000 // 1 GB/s default + } + // Parse the limit using the same format as network (e.g., "500MB/s") + capacity, err := parseDiskIOLimit(m.cfg.DiskIOLimit) + if err != nil { + return 1 * 1000 * 1000 * 1000 // 1 GB/s fallback + } + return capacity +} + +// DefaultNetworkBandwidth calculates the default network bandwidth for an instance +// based on its CPU allocation proportional to host CPU capacity. +// Formula: (instanceVcpus / hostCpuCapacity) * networkCapacity * oversubRatio +// Returns symmetric download/upload limits. +func (m *Manager) DefaultNetworkBandwidth(vcpus int) (downloadBps, uploadBps int64) { + cpuCapacity := m.CPUCapacity() + if cpuCapacity == 0 { + return 0, 0 + } + + netCapacity := m.NetworkCapacity() + if netCapacity == 0 { + return 0, 0 + } + + ratio := m.GetOversubRatio(ResourceNetwork) + effectiveNet := int64(float64(netCapacity) * ratio) + + // Proportional to CPU: (vcpus / cpuCapacity) * effectiveNet + bandwidth := (int64(vcpus) * effectiveNet) / cpuCapacity + + // Symmetric limits by default + return bandwidth, bandwidth +} + +// DefaultDiskIOBandwidth calculates the default disk I/O bandwidth for an instance +// based on its CPU allocation proportional to host CPU capacity. +// Formula: (instanceVcpus / hostCpuCapacity) * diskIOCapacity * oversubRatio +// Returns sustained rate and burst rate (4x sustained). +func (m *Manager) DefaultDiskIOBandwidth(vcpus int) (ioBps, burstBps int64) { + cpuCapacity := m.CPUCapacity() + if cpuCapacity == 0 { + return 0, 0 + } + + ioCapacity := m.DiskIOCapacity() + if ioCapacity == 0 { + return 0, 0 + } + + ratio := m.cfg.OversubDiskIO + if ratio <= 0 { + ratio = 2.0 // Default 2x oversubscription for disk I/O + } + effectiveIO := int64(float64(ioCapacity) * ratio) + + // Proportional to CPU: (vcpus / cpuCapacity) * effectiveIO + sustained := (int64(vcpus) * effectiveIO) / cpuCapacity + + // Burst is 4x sustained (allows fast cold starts) + burst := sustained * 4 + + return sustained, burst +} + +// HasSufficientDiskForPull checks if there's enough disk space for an image pull. +// Returns an error if available disk is below the minimum threshold (5GB). +func (m *Manager) HasSufficientDiskForPull(ctx context.Context) error { + const minDiskForPull = 5 * 1024 * 1024 * 1024 // 5GB + + status, err := m.GetStatus(ctx, ResourceDisk) + if err != nil { + return fmt.Errorf("check disk status: %w", err) + } + + if status.Available < minDiskForPull { + return fmt.Errorf("insufficient disk space for image pull: %d bytes available, minimum %d bytes required", + status.Available, minDiskForPull) + } + + return nil +} + +// MaxImageStorageBytes returns the maximum allowed image storage (OCI cache + rootfs). +// Based on MaxImageStorage fraction (default 20%) of disk capacity. +func (m *Manager) MaxImageStorageBytes() int64 { + m.mu.RLock() + diskRes, ok := m.resources[ResourceDisk] + m.mu.RUnlock() + + if !ok { + return 0 + } + + capacity := diskRes.Capacity() + fraction := m.cfg.MaxImageStorage + if fraction <= 0 { + fraction = 0.2 // Default 20% + } + + return int64(float64(capacity) * fraction) +} + +// CurrentImageStorageBytes returns the current image storage usage (OCI cache + rootfs). +func (m *Manager) CurrentImageStorageBytes(ctx context.Context) (int64, error) { + if m.imageLister == nil { + return 0, nil + } + + rootfsBytes, err := m.imageLister.TotalImageBytes(ctx) + if err != nil { + return 0, err + } + + ociCacheBytes, err := m.imageLister.TotalOCICacheBytes(ctx) + if err != nil { + return 0, err + } + + return rootfsBytes + ociCacheBytes, nil +} + +// HasSufficientImageStorage checks if pulling another image would exceed the image storage limit. +// Returns an error if current image storage >= max allowed. +func (m *Manager) HasSufficientImageStorage(ctx context.Context) error { + current, err := m.CurrentImageStorageBytes(ctx) + if err != nil { + return fmt.Errorf("check image storage: %w", err) + } + + max := m.MaxImageStorageBytes() + if max > 0 && current >= max { + return fmt.Errorf("image storage limit exceeded: %d bytes used, limit is %d bytes (%.0f%% of disk)", + current, max, m.cfg.MaxImageStorage*100) + } + + return nil +} diff --git a/lib/resources/resource_test.go b/lib/resources/resource_test.go new file mode 100644 index 0000000..2d4c08c --- /dev/null +++ b/lib/resources/resource_test.go @@ -0,0 +1,464 @@ +package resources + +import ( + "context" + "testing" + + "github.com/onkernel/hypeman/cmd/api/config" + "github.com/onkernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockInstanceLister implements InstanceLister for testing +type mockInstanceLister struct { + allocations []InstanceAllocation +} + +func (m *mockInstanceLister) ListInstanceAllocations(ctx context.Context) ([]InstanceAllocation, error) { + return m.allocations, nil +} + +// mockImageLister implements ImageLister for testing +type mockImageLister struct { + totalBytes int64 + ociCacheBytes int64 +} + +func (m *mockImageLister) TotalImageBytes(ctx context.Context) (int64, error) { + return m.totalBytes, nil +} + +func (m *mockImageLister) TotalOCICacheBytes(ctx context.Context) (int64, error) { + return m.ociCacheBytes, nil +} + +// mockVolumeLister implements VolumeLister for testing +type mockVolumeLister struct { + totalBytes int64 +} + +func (m *mockVolumeLister) TotalVolumeBytes(ctx context.Context) (int64, error) { + return m.totalBytes, nil +} + +func TestNewManager(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 2.0, + OversubMemory: 1.5, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + require.NotNil(t, mgr) +} + +func TestGetOversubRatio(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 2.0, + OversubMemory: 1.5, + OversubDisk: 1.0, + OversubNetwork: 3.0, + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + + assert.Equal(t, 2.0, mgr.GetOversubRatio(ResourceCPU)) + assert.Equal(t, 1.5, mgr.GetOversubRatio(ResourceMemory)) + assert.Equal(t, 1.0, mgr.GetOversubRatio(ResourceDisk)) + assert.Equal(t, 3.0, mgr.GetOversubRatio(ResourceNetwork)) + assert.Equal(t, 1.0, mgr.GetOversubRatio("unknown")) // default +} + +func TestDefaultNetworkBandwidth(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + NetworkLimit: "10Gbps", // 1.25 GB/s = 1,250,000,000 bytes/sec + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(&mockInstanceLister{}) + mgr.SetImageLister(&mockImageLister{}) + mgr.SetVolumeLister(&mockVolumeLister{}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + // With 10Gbps network and CPU capacity (varies by host) + // If host has 8 CPUs and instance wants 2, it gets 2/8 = 25% of network + cpuCapacity := mgr.CPUCapacity() + netCapacity := mgr.NetworkCapacity() + + if cpuCapacity > 0 && netCapacity > 0 { + // Request 2 vCPUs + downloadBw, uploadBw := mgr.DefaultNetworkBandwidth(2) + expected := (int64(2) * netCapacity) / cpuCapacity + assert.Equal(t, expected, downloadBw) + assert.Equal(t, expected, uploadBw) // Symmetric by default + } +} + +func TestDefaultNetworkBandwidth_ZeroCPU(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + // Don't initialize - CPU capacity will be 0 + + downloadBw, uploadBw := mgr.DefaultNetworkBandwidth(2) + assert.Equal(t, int64(0), downloadBw, "Should return 0 when CPU capacity is 0") + assert.Equal(t, int64(0), uploadBw, "Should return 0 when CPU capacity is 0") +} + +func TestParseNetworkLimit(t *testing.T) { + tests := []struct { + input string + expected int64 + wantErr bool + }{ + {"1Gbps", 125000000, false}, // 1 Gbps = 125 MB/s (decimal) + {"10Gbps", 1250000000, false}, // 10 Gbps = 1.25 GB/s (decimal) + {"100Mbps", 12500000, false}, // 100 Mbps = 12.5 MB/s (decimal) + {"1000kbps", 125000, false}, // 1000 kbps = 125 KB/s (decimal) + {"125MB", 125 * 1024 * 1024, false}, // 125 MiB (datasize uses binary) + {"1GB", 1024 * 1024 * 1024, false}, // 1 GiB (datasize uses binary) + {"invalid", 0, true}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result, err := parseNetworkLimit(tt.input) + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} + +func TestCPUResource_Capacity(t *testing.T) { + cpu, err := NewCPUResource() + require.NoError(t, err) + + // Should detect at least 1 CPU + assert.GreaterOrEqual(t, cpu.Capacity(), int64(1)) + assert.Equal(t, ResourceCPU, cpu.Type()) +} + +func TestMemoryResource_Capacity(t *testing.T) { + mem, err := NewMemoryResource() + require.NoError(t, err) + + // Should detect at least 1GB of memory + assert.GreaterOrEqual(t, mem.Capacity(), int64(1024*1024*1024)) + assert.Equal(t, ResourceMemory, mem.Type()) +} + +func TestCPUResource_Allocated(t *testing.T) { + cpu, err := NewCPUResource() + require.NoError(t, err) + + // With no instance lister, allocated should be 0 + allocated, err := cpu.Allocated(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(0), allocated) + + // With instance lister + cpu.SetInstanceLister(&mockInstanceLister{ + allocations: []InstanceAllocation{ + {ID: "1", Vcpus: 4, State: "Running"}, + {ID: "2", Vcpus: 2, State: "Paused"}, + {ID: "3", Vcpus: 8, State: "Stopped"}, // Not counted + }, + }) + + allocated, err = cpu.Allocated(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(6), allocated) // 4 + 2 = 6 (Stopped not counted) +} + +func TestMemoryResource_Allocated(t *testing.T) { + mem, err := NewMemoryResource() + require.NoError(t, err) + + mem.SetInstanceLister(&mockInstanceLister{ + allocations: []InstanceAllocation{ + {ID: "1", MemoryBytes: 4 * 1024 * 1024 * 1024, State: "Running"}, + {ID: "2", MemoryBytes: 2 * 1024 * 1024 * 1024, State: "Created"}, + {ID: "3", MemoryBytes: 8 * 1024 * 1024 * 1024, State: "Standby"}, // Not counted + }, + }) + + allocated, err := mem.Allocated(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(6*1024*1024*1024), allocated) +} + +func TestIsActiveState(t *testing.T) { + assert.True(t, isActiveState("Running")) + assert.True(t, isActiveState("Paused")) + assert.True(t, isActiveState("Created")) + assert.False(t, isActiveState("Stopped")) + assert.False(t, isActiveState("Standby")) + assert.False(t, isActiveState("Unknown")) +} + +func TestHasSufficientDiskForPull(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(&mockInstanceLister{}) + mgr.SetImageLister(&mockImageLister{}) + mgr.SetVolumeLister(&mockVolumeLister{}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + // This test depends on actual disk space available + // We just verify it doesn't error + err = mgr.HasSufficientDiskForPull(context.Background()) + // May or may not error depending on disk space - just verify it runs + _ = err +} + +// TestInitialize_SetsInstanceListersForAllResources verifies that Initialize +// properly propagates the instance lister to CPU and Memory resources. +// This catches a bug where CPU/Memory SetInstanceLister was not being called. +func TestInitialize_SetsInstanceListersForAllResources(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + // Create a mock lister that returns known allocations + mockLister := &mockInstanceLister{ + allocations: []InstanceAllocation{ + { + ID: "test-1", + Vcpus: 4, + MemoryBytes: 8 * 1024 * 1024 * 1024, // 8GB + State: "Running", + }, + { + ID: "test-2", + Vcpus: 2, + MemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB + State: "Running", + }, + }, + } + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(mockLister) + mgr.SetImageLister(&mockImageLister{}) + mgr.SetVolumeLister(&mockVolumeLister{}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + // CPU should report correct allocation (4 + 2 = 6 vCPUs) + cpuStatus, err := mgr.GetStatus(context.Background(), ResourceCPU) + require.NoError(t, err) + assert.Equal(t, int64(6), cpuStatus.Allocated, "CPU should report 6 vCPUs allocated") + + // Memory should report correct allocation (8GB + 4GB = 12GB) + memStatus, err := mgr.GetStatus(context.Background(), ResourceMemory) + require.NoError(t, err) + assert.Equal(t, int64(12*1024*1024*1024), memStatus.Allocated, "Memory should report 12GB allocated") +} + +// TestGetFullStatus_ReturnsAllResourceAllocations verifies that GetFullStatus +// returns correct allocations for all resource types. +func TestGetFullStatus_ReturnsAllResourceAllocations(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 2.0, + OversubMemory: 1.5, + OversubDisk: 1.0, + OversubNetwork: 1.0, + NetworkLimit: "10Gbps", + } + p := paths.New(cfg.DataDir) + + mockLister := &mockInstanceLister{ + allocations: []InstanceAllocation{ + { + ID: "vm-1", + Name: "test-vm", + Vcpus: 4, + MemoryBytes: 8 * 1024 * 1024 * 1024, + OverlayBytes: 10 * 1024 * 1024 * 1024, + NetworkDownloadBps: 125000000, + NetworkUploadBps: 125000000, + State: "Running", + }, + }, + } + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(mockLister) + mgr.SetImageLister(&mockImageLister{totalBytes: 50 * 1024 * 1024 * 1024}) + mgr.SetVolumeLister(&mockVolumeLister{totalBytes: 100 * 1024 * 1024 * 1024}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + status, err := mgr.GetFullStatus(context.Background()) + require.NoError(t, err) + + // Verify CPU status + assert.Equal(t, int64(4), status.CPU.Allocated) + assert.Equal(t, 2.0, status.CPU.OversubRatio) + + // Verify Memory status + assert.Equal(t, int64(8*1024*1024*1024), status.Memory.Allocated) + assert.Equal(t, 1.5, status.Memory.OversubRatio) + + // Verify allocations list + require.Len(t, status.Allocations, 1) + assert.Equal(t, "vm-1", status.Allocations[0].InstanceID) + assert.Equal(t, 4, status.Allocations[0].CPU) + assert.Equal(t, int64(8*1024*1024*1024), status.Allocations[0].MemoryBytes) +} + +// TestNetworkResource_Allocated verifies network allocation tracking +// uses max(download, upload) since they share the physical link. +func TestNetworkResource_Allocated(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + NetworkLimit: "1Gbps", // 125MB/s + OversubNetwork: 1.0, + } + + mockLister := &mockInstanceLister{ + allocations: []InstanceAllocation{ + {ID: "1", NetworkDownloadBps: 50000000, NetworkUploadBps: 30000000, State: "Running"}, // max = 50MB/s + {ID: "2", NetworkDownloadBps: 20000000, NetworkUploadBps: 40000000, State: "Running"}, // max = 40MB/s + {ID: "3", NetworkDownloadBps: 10000000, NetworkUploadBps: 10000000, State: "Stopped"}, // Not counted + }, + } + + net, err := NewNetworkResource(cfg, mockLister) + require.NoError(t, err) + + allocated, err := net.Allocated(context.Background()) + require.NoError(t, err) + // Should be 50MB/s + 40MB/s = 90MB/s + assert.Equal(t, int64(90000000), allocated) +} + +// TestMaxImageStorage verifies the image storage limit calculation +func TestMaxImageStorage(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + MaxImageStorage: 0.2, // 20% + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(&mockInstanceLister{}) + mgr.SetImageLister(&mockImageLister{ + totalBytes: 50 * 1024 * 1024 * 1024, // 50GB rootfs + ociCacheBytes: 25 * 1024 * 1024 * 1024, // 25GB OCI cache + }) + mgr.SetVolumeLister(&mockVolumeLister{}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + // Max should be 20% of disk capacity + maxBytes := mgr.MaxImageStorageBytes() + diskCapacity := mgr.resources[ResourceDisk].Capacity() + expectedMax := int64(float64(diskCapacity) * 0.2) + assert.Equal(t, expectedMax, maxBytes) + + // Current should be 50GB + 25GB = 75GB + currentBytes, err := mgr.CurrentImageStorageBytes(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(75*1024*1024*1024), currentBytes) +} + +// TestDiskBreakdown_IncludesOCICacheAndVolumeOverlays verifies disk breakdown +// includes OCI cache and volume overlays +func TestDiskBreakdown_IncludesOCICacheAndVolumeOverlays(t *testing.T) { + cfg := &config.Config{ + DataDir: t.TempDir(), + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } + p := paths.New(cfg.DataDir) + + mockInstances := &mockInstanceLister{ + allocations: []InstanceAllocation{ + { + ID: "vm-1", + OverlayBytes: 10 * 1024 * 1024 * 1024, // 10GB rootfs overlay + VolumeOverlayBytes: 5 * 1024 * 1024 * 1024, // 5GB volume overlays + State: "Running", + }, + { + ID: "vm-2", + OverlayBytes: 8 * 1024 * 1024 * 1024, // 8GB rootfs overlay + VolumeOverlayBytes: 2 * 1024 * 1024 * 1024, // 2GB volume overlays + State: "Running", + }, + }, + } + + mgr := NewManager(cfg, p) + mgr.SetInstanceLister(mockInstances) + mgr.SetImageLister(&mockImageLister{ + totalBytes: 50 * 1024 * 1024 * 1024, // 50GB rootfs + ociCacheBytes: 25 * 1024 * 1024 * 1024, // 25GB OCI cache + }) + mgr.SetVolumeLister(&mockVolumeLister{totalBytes: 100 * 1024 * 1024 * 1024}) + + err := mgr.Initialize(context.Background()) + require.NoError(t, err) + + status, err := mgr.GetFullStatus(context.Background()) + require.NoError(t, err) + require.NotNil(t, status.DiskDetail) + + // Verify breakdown + assert.Equal(t, int64(50*1024*1024*1024), status.DiskDetail.Images) + assert.Equal(t, int64(25*1024*1024*1024), status.DiskDetail.OCICache) + assert.Equal(t, int64(100*1024*1024*1024), status.DiskDetail.Volumes) + // Overlays should be (10+5) + (8+2) = 25GB + assert.Equal(t, int64(25*1024*1024*1024), status.DiskDetail.Overlays) +} diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index d3a3dfc..9e254f7 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -33,6 +33,10 @@ type Manager interface { // GetVolumePath returns the path to the volume data file GetVolumePath(id string) string + + // TotalVolumeBytes returns the total size of all volumes. + // Used by the resource manager for disk capacity tracking. + TotalVolumeBytes(ctx context.Context) (int64, error) } type manager struct { @@ -393,6 +397,11 @@ func (m *manager) GetVolumePath(id string) string { return m.paths.VolumeData(id) } +// TotalVolumeBytes returns the total size of all volumes. +func (m *manager) TotalVolumeBytes(ctx context.Context) (int64, error) { + return m.calculateTotalVolumeStorage(ctx) +} + // metadataToVolume converts stored metadata to a Volume struct func (m *manager) metadataToVolume(meta *storedMetadata) *Volume { createdAt, _ := time.Parse(time.RFC3339, meta.CreatedAt) diff --git a/openapi.yaml b/openapi.yaml index c5d48db..fac72d3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -127,6 +127,10 @@ components: description: Writable overlay disk size (human-readable format like "10GB", "50G") default: "10GB" example: "20GB" + disk_io_bps: + type: string + description: Disk I/O rate limit (e.g., "100MB/s", "500MB/s"). Defaults to proportional share based on CPU allocation if configured. + example: "100MB/s" vcpus: type: integer description: Number of virtual CPUs @@ -149,6 +153,14 @@ components: description: Whether to attach instance to the default network default: true example: true + bandwidth_download: + type: string + description: Download bandwidth limit (external→VM, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. + example: "1Gbps" + bandwidth_upload: + type: string + description: Upload bandwidth limit (VM→external, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. + example: "1Gbps" devices: type: array items: @@ -206,6 +218,10 @@ components: type: integer description: Number of virtual CPUs example: 2 + disk_io_bps: + type: string + description: Disk I/O rate limit (human-readable, e.g., "100MB/s") + example: "100MB/s" env: type: object additionalProperties: @@ -233,6 +249,14 @@ components: description: Assigned MAC address (null if no network) example: "02:00:00:ab:cd:ef" nullable: true + bandwidth_download: + type: string + description: Download bandwidth limit (human-readable, e.g., "1Gbps", "125MB/s") + example: "125MB/s" + bandwidth_upload: + type: string + description: Upload bandwidth limit (human-readable, e.g., "1Gbps", "125MB/s") + example: "125MB/s" volumes: type: array description: Volumes attached to the instance @@ -659,6 +683,123 @@ components: nullable: true example: "nvidia" + ResourceStatus: + type: object + required: [type, capacity, effective_limit, allocated, available, oversub_ratio] + properties: + type: + type: string + description: Resource type + example: "cpu" + capacity: + type: integer + format: int64 + description: Raw host capacity + example: 64 + effective_limit: + type: integer + format: int64 + description: Capacity after oversubscription (capacity * ratio) + example: 128 + allocated: + type: integer + format: int64 + description: Currently allocated resources + example: 48 + available: + type: integer + format: int64 + description: Available for allocation (effective_limit - allocated) + example: 80 + oversub_ratio: + type: number + format: double + description: Oversubscription ratio applied + example: 2.0 + source: + type: string + description: How capacity was determined (detected, configured) + example: "detected" + + DiskBreakdown: + type: object + properties: + images_bytes: + type: integer + format: int64 + description: Disk used by exported rootfs images + example: 214748364800 + oci_cache_bytes: + type: integer + format: int64 + description: Disk used by OCI layer cache (shared blobs) + example: 53687091200 + volumes_bytes: + type: integer + format: int64 + description: Disk used by volumes + example: 107374182400 + overlays_bytes: + type: integer + format: int64 + description: Disk used by instance overlays (rootfs + volume overlays) + example: 227633306624 + + ResourceAllocation: + type: object + properties: + instance_id: + type: string + description: Instance identifier + example: "abc123" + instance_name: + type: string + description: Instance name + example: "my-vm" + cpu: + type: integer + description: vCPUs allocated + example: 4 + memory_bytes: + type: integer + format: int64 + description: Memory allocated in bytes + example: 8589934592 + disk_bytes: + type: integer + format: int64 + description: Disk allocated in bytes (overlay + volumes) + example: 10737418240 + network_download_bps: + type: integer + format: int64 + description: Download bandwidth limit in bytes/sec (external→VM) + example: 125000000 + network_upload_bps: + type: integer + format: int64 + description: Upload bandwidth limit in bytes/sec (VM→external) + example: 125000000 + + Resources: + type: object + required: [cpu, memory, disk, network, allocations] + properties: + cpu: + $ref: "#/components/schemas/ResourceStatus" + memory: + $ref: "#/components/schemas/ResourceStatus" + disk: + $ref: "#/components/schemas/ResourceStatus" + network: + $ref: "#/components/schemas/ResourceStatus" + disk_breakdown: + $ref: "#/components/schemas/DiskBreakdown" + allocations: + type: array + items: + $ref: "#/components/schemas/ResourceAllocation" + paths: /health: get: @@ -672,6 +813,29 @@ paths: schema: $ref: "#/components/schemas/Health" + /resources: + get: + summary: Get host resource capacity and allocations + description: | + Returns current host resource capacity, allocation status, and per-instance breakdown. + Resources include CPU, memory, disk, and network. Oversubscription ratios are applied + to calculate effective limits. + operationId: getResources + security: + - bearerAuth: [] + responses: + 200: + description: Resource status + content: + application/json: + schema: + $ref: "#/components/schemas/Resources" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /images: get: