diff --git a/test/kubernetes/benchmarks/BUILD b/test/kubernetes/benchmarks/BUILD index 7754ed7140..9994699642 100644 --- a/test/kubernetes/benchmarks/BUILD +++ b/test/kubernetes/benchmarks/BUILD @@ -75,6 +75,7 @@ go_library( "//test/kubernetes/k8sctx", "//test/kubernetes/testcluster", "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/resource:go_default_library", "@org_golang_x_time//rate:go_default_library", ], ) diff --git a/test/kubernetes/benchmarks/abslbuild.go b/test/kubernetes/benchmarks/abslbuild.go index 16684ce785..df2ac45a4c 100644 --- a/test/kubernetes/benchmarks/abslbuild.go +++ b/test/kubernetes/benchmarks/abslbuild.go @@ -65,10 +65,15 @@ func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { t.Fatalf("Failed to resolve image: %v", err) } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } for _, test := range []struct { - name string - volume *v13.Volume + name string + volume *v13.Volume + disabled bool }{ { name: "RootFS", @@ -93,9 +98,13 @@ func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex }, }, }, + disabled: !supportsPV, }, } { t.Run(test.name, func(t *testing.T) { + if test.disabled { + t.Skip("Persistent volumes are not supported in this cluster") + } endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) diff --git a/test/kubernetes/benchmarks/ffmpeg.go b/test/kubernetes/benchmarks/ffmpeg.go index 2aa6f2a0dd..519ce2edd6 100644 --- a/test/kubernetes/benchmarks/ffmpeg.go +++ b/test/kubernetes/benchmarks/ffmpeg.go @@ -69,10 +69,15 @@ func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { t.Fatalf("Failed to resolve image: %v", err) } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } for _, test := range []struct { - name string - volume *v13.Volume + name string + volume *v13.Volume + disabled bool }{ { name: "RootFS", @@ -97,9 +102,13 @@ func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex }, }, }, + disabled: !supportsPV, }, } { t.Run(test.name, func(t *testing.T) { + if test.disabled { + t.Skip("Persistent volumes are not supported in this cluster") + } endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) diff --git a/test/kubernetes/benchmarks/grpc.go b/test/kubernetes/benchmarks/grpc.go index 537165aadc..5573569dbc 100644 --- a/test/kubernetes/benchmarks/grpc.go +++ b/test/kubernetes/benchmarks/grpc.go @@ -70,10 +70,15 @@ func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { t.Fatalf("Failed to resolve image: %v", err) } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } for _, test := range []struct { - name string - volume *v13.Volume + name string + volume *v13.Volume + disabled bool }{ { name: "RootFS", @@ -98,9 +103,13 @@ func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex }, }, }, + disabled: !supportsPV, }, } { t.Run(test.name, func(t *testing.T) { + if test.disabled { + t.Skip("Persistent volumes are not supported in this cluster") + } endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) diff --git a/test/kubernetes/benchmarks/gsutil.go b/test/kubernetes/benchmarks/gsutil.go index 92ed4969fd..c22fcfa53a 100644 --- a/test/kubernetes/benchmarks/gsutil.go +++ b/test/kubernetes/benchmarks/gsutil.go @@ -71,12 +71,17 @@ func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { t.Fatalf("Failed to resolve image: %v", err) } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } // Run tests with different volume types. We could also use gsutil // parallel sliced downloads as a test dimension. for _, storage := range []struct { - name string - volume *v13.Volume + name string + volume *v13.Volume + disabled bool }{ { name: "RootFS", @@ -101,9 +106,13 @@ func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex }, }, }, + disabled: !supportsPV, }, } { t.Run(storage.name, func(t *testing.T) { + if storage.disabled { + t.Skip("Persistent volumes are not supported in this cluster") + } for _, slicing := range []struct { name string option string diff --git a/test/kubernetes/benchmarks/nginx.go b/test/kubernetes/benchmarks/nginx.go index 30bfecc87e..ffd7d6e286 100644 --- a/test/kubernetes/benchmarks/nginx.go +++ b/test/kubernetes/benchmarks/nginx.go @@ -78,21 +78,12 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC if nginxImage, err = k8sCtx.ResolveImage(ctx, nginxImage); err != nil { t.Fatalf("Failed to resolve image: %v", err) } - - persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) - if err != nil { - t.Fatalf("Failed to create persistent volume: %v", err) - } - defer cluster.DeletePersistentVolume(ctx, persistentVol) - - for _, test := range []struct { - // Name of the test. - name string - // Suffix for pod names, must be short enough. + type nginxTest struct { + name string suffix string - // Volume to mount at /tmp/root. volume *v13.Volume - }{ + } + tests := []nginxTest{ { name: "RootFS", suffix: "rootfs", @@ -108,7 +99,18 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC }, }, }, - { + } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } + if supportsPV { + persistentVol, err := cluster.CreatePersistentVolume(ctx, benchmarkNS.GetPersistentVolume("nginx-data", "30Gi")) + if err != nil { + t.Fatalf("Failed to create persistent volume: %v", err) + } + defer cluster.DeletePersistentVolume(ctx, persistentVol) + tests = append(tests, nginxTest{ name: "PersistentVolume", suffix: "pvol", volume: &v13.Volume{ @@ -119,8 +121,10 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC }, }, }, - }, - } { + }) + } + + for _, test := range tests { t.Run(test.name, func(t *testing.T) { endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { diff --git a/test/kubernetes/benchmarks/rubydev.go b/test/kubernetes/benchmarks/rubydev.go index 2261aff884..65026b23ca 100644 --- a/test/kubernetes/benchmarks/rubydev.go +++ b/test/kubernetes/benchmarks/rubydev.go @@ -75,9 +75,15 @@ func RunRubyDev(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesConte if image, err = k8sCtx.ResolveImage(ctx, image); err != nil { t.Fatalf("failed to resolve image: %v", err) } + supportsPV, err := cluster.SupportsPersistentVolumes(ctx) + if err != nil { + t.Fatalf("Failed to check if cluster supports persistent volumes: %v", err) + } + for _, test := range []struct { - name string - volume *v13.Volume + name string + volume *v13.Volume + disabled bool }{ { name: "RootFS", @@ -102,9 +108,13 @@ func RunRubyDev(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesConte }, }, }, + disabled: !supportsPV, }, } { t.Run(test.name, func(t *testing.T) { + if test.disabled { + t.Skip("Persistent volumes are not supported in this cluster") + } endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS) if err != nil { t.Fatalf("Failed to setup profiling: %v", err) diff --git a/test/kubernetes/benchmarks/startup.go b/test/kubernetes/benchmarks/startup.go index e8cef13ea1..0b4ffed18f 100644 --- a/test/kubernetes/benchmarks/startup.go +++ b/test/kubernetes/benchmarks/startup.go @@ -32,6 +32,7 @@ import ( "gvisor.dev/gvisor/test/kubernetes/k8sctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" v13 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) const ( @@ -183,6 +184,12 @@ func runPodAndGetMetrics(ctx context.Context, t *testing.T, k8sCtx k8sctx.Kubern if err != nil { return nil, fmt.Errorf("failed to set pod for test nodepool: %w", err) } + // If the pod has explicit limits set as a result of the runtime choice, + // clip them to a tiny size. This is because for runtimes that require + // explicit limits (e.g. Kata), initialization may scale together with + // the limits, which isn't the point of this benchmark; we want to measure + // the runtime of starting a container with minimal footprint. + maybeClipResources(p) createdAt := time.Now() p, err = cluster.CreatePod(ctx, p) if err != nil { @@ -404,3 +411,31 @@ func runPodAndGetMetrics(ctx context.Context, t *testing.T, k8sCtx k8sctx.Kubern deleted = true return metrics, nil } + +func maybeClipResources(p *v13.Pod) { + // Same size as e2-medium. + const ( + cpuLimit = "2" + memoryLimit = "4Gi" + ) + for _, containers := range [][]v13.Container{ + p.Spec.InitContainers, + p.Spec.Containers, + } { + for i := range containers { + for _, resources := range []v13.ResourceList{ + containers[i].Resources.Limits, + containers[i].Resources.Requests, + } { + if resources != nil { + if _, ok := resources[v13.ResourceCPU]; ok { + resources[v13.ResourceCPU] = resource.MustParse(cpuLimit) + } + if _, ok := resources[v13.ResourceMemory]; ok { + resources[v13.ResourceMemory] = resource.MustParse(memoryLimit) + } + } + } + } + } +} diff --git a/test/kubernetes/testcluster/objects.go b/test/kubernetes/testcluster/objects.go index e9e36bb923..0657a7dfe0 100644 --- a/test/kubernetes/testcluster/objects.go +++ b/test/kubernetes/testcluster/objects.go @@ -187,51 +187,18 @@ func (n *Namespace) GetService(name string, spec v13.ServiceSpec) *v13.Service { // ContainerResourcesRequest holds arguments to set requested resource on a container. type ContainerResourcesRequest struct { - CPUResources string // CPUResources to request. Note: Will be overridden by flag above. - MemoryResources string // MemoryResources to request. Note: Will be overridden by flag above. - GPU bool - TPU bool + GPU bool + TPU bool } // String returns a string representation of the `ContainerResourcesRequest`. func (crr ContainerResourcesRequest) String() string { - return fmt.Sprintf("cpu=%q memory=%q gpu=%v tpu=%v", crr.CPUResources, crr.MemoryResources, crr.GPU, crr.TPU) + return fmt.Sprintf("gpu=%v tpu=%v", crr.GPU, crr.TPU) } // SetContainerResources sets container resources. -// Sets both the resource limits and requests as container runtimes honor -// them differently. // `containerName` is optional if the pod has exactly one container. func SetContainerResources(pod *v13.Pod, containerName string, requests ContainerResourcesRequest) (*v13.Pod, error) { - resourceList := v13.ResourceList{} - if requests.CPUResources != "" { - resourceList[v13.ResourceCPU] = resource.MustParse(requests.CPUResources) - } - if requests.MemoryResources != "" { - resourceList[v13.ResourceMemory] = resource.MustParse(requests.MemoryResources) - } - - if requests.GPU { - acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolNumAcceleratorsKey] - if !ok { - return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") - } - resourceList[v13.ResourceName("nvidia.com/gpu")] = resource.MustParse(acceleratorCount) - } - - if requests.TPU { - acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolTPUNumAcceleratorKey] - if !ok { - return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") - } - resourceList[v13.ResourceName("google.com/tpu")] = resource.MustParse(acceleratorCount) - } - - requirements := v13.ResourceRequirements{ - Limits: resourceList, - Requests: resourceList, - } - var containerToChange *v13.Container if containerName == "" { switch len(pod.Spec.Containers) { @@ -252,7 +219,25 @@ func SetContainerResources(pod *v13.Pod, containerName string, requests Containe if containerToChange == nil { return nil, fmt.Errorf("container %q not found", containerName) } - containerToChange.Resources = requirements + for _, resourceList := range []v13.ResourceList{ + containerToChange.Resources.Limits, + containerToChange.Resources.Requests, + } { + if requests.GPU { + acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolNumAcceleratorsKey] + if !ok { + return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") + } + resourceList[v13.ResourceName("nvidia.com/gpu")] = resource.MustParse(acceleratorCount) + } + if requests.TPU { + acceleratorCount, ok := pod.Spec.NodeSelector[NodepoolTPUNumAcceleratorKey] + if !ok { + return nil, fmt.Errorf("cannot determine number of accelerators that the pod should use, make sure to call ConfigurePodForRuntimeTestNodepool first") + } + resourceList[v13.ResourceName("google.com/tpu")] = resource.MustParse(acceleratorCount) + } + } return pod, nil } @@ -376,19 +361,19 @@ type RuntimeType string // List of known runtime types. const ( - RuntimeTypeGVisor = RuntimeType("gvisor") - RuntimeTypeUnsandboxed = RuntimeType("runc") - RuntimeTypeGVisorTPU = RuntimeType("gvisor-tpu") - RuntimeTypeUnsandboxedTPU = RuntimeType("runc-tpu") - RuntimeTypeNestedKataQEMU = RuntimeType("nested-kata-qemu") - RuntimeTypeNestedKataCloudHypervisor = RuntimeType("nested-kata-cloudhypervisor") - RuntimeTypeNestedKataFirecracker = RuntimeType("nested-kata-firecracker") + RuntimeTypeGVisor = RuntimeType("gvisor") + RuntimeTypeUnsandboxed = RuntimeType("runc") + RuntimeTypeGVisorTPU = RuntimeType("gvisor-tpu") + RuntimeTypeUnsandboxedTPU = RuntimeType("runc-tpu") + RuntimeTypeKataQEMU = RuntimeType("kata-qemu") + RuntimeTypeKataCloudHypervisor = RuntimeType("kata-cloudhypervisor") + RuntimeTypeKataFirecracker = RuntimeType("kata-firecracker") ) // IsValid returns true if the runtime type is valid. func (t RuntimeType) IsValid() bool { switch t { - case RuntimeTypeGVisor, RuntimeTypeUnsandboxed, RuntimeTypeGVisorTPU, RuntimeTypeUnsandboxedTPU, RuntimeTypeNestedKataQEMU, RuntimeTypeNestedKataCloudHypervisor, RuntimeTypeNestedKataFirecracker: + case RuntimeTypeGVisor, RuntimeTypeUnsandboxed, RuntimeTypeGVisorTPU, RuntimeTypeUnsandboxedTPU, RuntimeTypeKataQEMU, RuntimeTypeKataCloudHypervisor, RuntimeTypeKataFirecracker: return true default: return false @@ -402,17 +387,17 @@ func (t RuntimeType) IsGVisor() bool { // IsKata returns true if the runtime is a Kata-based runtime. func (t RuntimeType) IsKata() bool { - return t == RuntimeTypeNestedKataQEMU || t == RuntimeTypeNestedKataCloudHypervisor || t == RuntimeTypeNestedKataFirecracker + return t == RuntimeTypeKataQEMU || t == RuntimeTypeKataCloudHypervisor || t == RuntimeTypeKataFirecracker } // KataShimName returns the Kata shim name for the runtime type. func (t RuntimeType) KataShimName() (string, error) { switch t { - case RuntimeTypeNestedKataQEMU: + case RuntimeTypeKataQEMU: return "kata-qemu", nil - case RuntimeTypeNestedKataCloudHypervisor: + case RuntimeTypeKataCloudHypervisor: return "kata-clh", nil - case RuntimeTypeNestedKataFirecracker: + case RuntimeTypeKataFirecracker: return "kata-fc", nil default: return "", fmt.Errorf("not a Kata runtime: %q", t) @@ -426,6 +411,20 @@ func (t RuntimeType) RequiresExplicitResourceLimits() bool { return t.IsKata() } +// MaxCores returns the maximum number of cores that can be used by a pod using this runtime. +// Returns 0 if there is no pod core limit. +func (t RuntimeType) MaxCores() int { + switch t { + case RuntimeTypeKataFirecracker: + // Firecracker only supports up to 32 vCPUs: + // https://github.com/firecracker-microvm/firecracker/blob/e865b9c5fad47384c15a6c57c6c6e628210f2282/src/vmm/src/vmm_config/machine_config.rs#L11-L13 + // Experimentally, using 32 still fails, but 31 works. + return 31 + default: + return 0 + } +} + // ApplyNodepool modifies the nodepool to configure it to use the runtime. func (t RuntimeType) ApplyNodepool(nodepool *cspb.NodePool) { if nodepool.GetConfig().GetLabels() == nil { @@ -451,12 +450,12 @@ func (t RuntimeType) ApplyNodepool(nodepool *cspb.NodePool) { }) case RuntimeTypeUnsandboxedTPU: nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeUnsandboxedTPU) - case RuntimeTypeNestedKataQEMU: - nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeNestedKataQEMU) - case RuntimeTypeNestedKataCloudHypervisor: - nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeNestedKataCloudHypervisor) - case RuntimeTypeNestedKataFirecracker: - nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeNestedKataFirecracker) + case RuntimeTypeKataQEMU: + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeKataQEMU) + case RuntimeTypeKataCloudHypervisor: + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeKataCloudHypervisor) + case RuntimeTypeKataFirecracker: + nodepool.Config.Labels[NodepoolRuntimeKey] = string(RuntimeTypeKataFirecracker) default: panic(fmt.Sprintf("unsupported runtime %q", t)) } @@ -544,7 +543,7 @@ func (t RuntimeType) ApplyPodSpec(podSpec *v13.PodSpec) { Operator: v13.TolerationOpEqual, Value: gvisorRuntimeClass, }) - case RuntimeTypeNestedKataQEMU, RuntimeTypeNestedKataCloudHypervisor, RuntimeTypeNestedKataFirecracker: + case RuntimeTypeKataQEMU, RuntimeTypeKataCloudHypervisor, RuntimeTypeKataFirecracker: shimName, err := t.KataShimName() if err != nil { // Logically impossible since we already checked that t is a Kata runtime, so panic is appropriate. diff --git a/test/kubernetes/testcluster/objects_test.go b/test/kubernetes/testcluster/objects_test.go index 9f0898f484..d11cb8e1be 100644 --- a/test/kubernetes/testcluster/objects_test.go +++ b/test/kubernetes/testcluster/objects_test.go @@ -15,6 +15,7 @@ package testcluster import ( + "strings" "testing" v23 "k8s.io/api/core/v1" @@ -53,20 +54,302 @@ func TestGetPersistentVolume(t *testing.T) { } // Verify size. - requests := pvc.Spec.Resources.Requests - if requests == nil { - t.Fatalf("expected Spec.Resources.Requests to be set, but it was nil") - } + verifyResource(t, pvc.Spec.Resources.Requests, v23.ResourceStorage, tc.size) + }) + } +} - storage, ok := requests[v23.ResourceStorage] - if !ok { - t.Fatalf("expected storage request to be set") - } +func TestSetContainerResources(t *testing.T) { + testCases := []struct { + name string + pod *v23.Pod + containerName string + requests ContainerResourcesRequest + wantErr string + wantPod func(t *testing.T, got *v23.Pod) + }{ + { + name: "empty container name, single container, empty requests", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{}, + wantPod: func(t *testing.T, got *v23.Pod) { + if got == nil { + t.Fatal("got nil pod") + } + if len(got.Spec.Containers) != 1 { + t.Fatalf("expected 1 container, got %d", len(got.Spec.Containers)) + } + c := got.Spec.Containers[0] + if len(c.Resources.Limits) != 0 || len(c.Resources.Requests) != 0 { + t.Errorf("expected no resources set, got limits: %v, requests: %v", c.Resources.Limits, c.Resources.Requests) + } + }, + }, + { + name: "empty container name, multiple containers", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + {Name: "container-1"}, + {Name: "container-2"}, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{}, + wantErr: "multiple containers found in pod", + }, + { + name: "empty container name, no containers", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{}, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{}, + wantErr: "no containers found in pod", + }, + { + name: "non-empty container name, selects correct container", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + { + Name: "container-2", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "container-2", + requests: ContainerResourcesRequest{}, + wantPod: func(t *testing.T, got *v23.Pod) { + if got == nil { + t.Fatal("got nil pod") + } + if got.Spec.Containers[0].Name != "container-1" || got.Spec.Containers[1].Name != "container-2" { + t.Errorf("unexpected container order") + } + }, + }, + { + name: "container name not found", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + {Name: "container-1"}, + }, + }, + }, + containerName: "container-2", + requests: ContainerResourcesRequest{}, + wantErr: `container "container-2" not found`, + }, + { + name: "GPU request without node selector", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{ + GPU: true, + }, + wantErr: "cannot determine number of accelerators", + }, + { + name: "TPU request without node selector", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{ + TPU: true, + }, + wantErr: "cannot determine number of accelerators", + }, + { + name: "GPU request with node selector key", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + NodeSelector: map[string]string{ + "num-accelerators": "4", + }, + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{ + GPU: true, + }, + wantPod: func(t *testing.T, got *v23.Pod) { + if got == nil { + t.Fatal("got nil pod") + } + c := got.Spec.Containers[0] + verifyResource(t, c.Resources.Limits, "nvidia.com/gpu", "4") + verifyResource(t, c.Resources.Requests, "nvidia.com/gpu", "4") + }, + }, + { + name: "TPU request with node selector key", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + NodeSelector: map[string]string{ + "cloud.google.com/gke-accelerator-count": "8", + }, + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{}, + Requests: v23.ResourceList{}, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{ + TPU: true, + }, + wantPod: func(t *testing.T, got *v23.Pod) { + if got == nil { + t.Fatal("got nil pod") + } + c := got.Spec.Containers[0] + verifyResource(t, c.Resources.Limits, "google.com/tpu", "8") + verifyResource(t, c.Resources.Requests, "google.com/tpu", "8") + }, + }, + { + name: "preserves existing CPU and memory limits and requests", + pod: &v23.Pod{ + Spec: v23.PodSpec{ + NodeSelector: map[string]string{ + "num-accelerators": "4", + }, + Containers: []v23.Container{ + { + Name: "container-1", + Resources: v23.ResourceRequirements{ + Limits: v23.ResourceList{ + v23.ResourceCPU: resource.MustParse("1"), + v23.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: v23.ResourceList{ + v23.ResourceCPU: resource.MustParse("500m"), + v23.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + containerName: "", + requests: ContainerResourcesRequest{ + GPU: true, + }, + wantPod: func(t *testing.T, got *v23.Pod) { + if got == nil { + t.Fatal("got nil pod") + } + c := got.Spec.Containers[0] + // Verify CPU and Memory are preserved. + verifyResource(t, c.Resources.Limits, v23.ResourceCPU, "1") + verifyResource(t, c.Resources.Limits, v23.ResourceMemory, "2Gi") + verifyResource(t, c.Resources.Requests, v23.ResourceCPU, "500m") + verifyResource(t, c.Resources.Requests, v23.ResourceMemory, "1Gi") + }, + }, + } - expectedSize := resource.MustParse(tc.size) - if !storage.Equal(expectedSize) { - t.Errorf("expected storage size %v, got %v", expectedSize, storage) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := SetContainerResources(tc.pod, tc.containerName, tc.requests) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("expected error containing %q, got %v", tc.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tc.wantPod != nil { + tc.wantPod(t, got) } }) } } + +func verifyResource(t *testing.T, list v23.ResourceList, name v23.ResourceName, want string) { + t.Helper() + if list == nil { + t.Errorf("resource list is nil, expected resource %q to be %q", name, want) + return + } + qty, ok := list[name] + if !ok { + t.Errorf("expected resource %q to be set", name) + return + } + expected := resource.MustParse(want) + if !qty.Equal(expected) { + t.Errorf("expected resource %q to be %v, got %v", name, expected, qty) + } +} diff --git a/test/kubernetes/testcluster/testcluster.go b/test/kubernetes/testcluster/testcluster.go index 0e067979db..cb233b60e0 100644 --- a/test/kubernetes/testcluster/testcluster.go +++ b/test/kubernetes/testcluster/testcluster.go @@ -166,17 +166,36 @@ type MachineInfo struct { // IsVirtual is whether the machine type is a virtual machine. IsVirtual bool + + // MaxPodCores is the maximum number of cores to set in a pod. + // If set to 0, but the runtime requires explicit limits, the + // the default value is based on `defaultMaxResourceUtilization`. + MaxPodCores int + + // MaxPodMemoryGiB is the maximum amount of memory in GiB to set in a pod. + // If set to 0, but the runtime requires explicit limits, the + // the default value is based on `defaultMaxResourceUtilization`. + MaxPodMemoryGiB int } +// defaultMaxResourceUtilization is the default maximum resource utilization for a machine type. +const defaultMaxResourceUtilization = 0.8 + // KnownMachineTypes is a map of known GCE machine types to their info. var KnownMachineTypes = map[string]*MachineInfo{ - "n1-standard-4": {NumCores: 4, MemoryGiB: 15, IsVirtual: true}, - "n2-standard-4": {NumCores: 4, MemoryGiB: 16, IsVirtual: true}, - "n2-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, - "n2d-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, - "g2-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, - "ct4p-hightpu-4t": {NumCores: 240, MemoryGiB: 407, IsVirtual: true}, - "c3-standard-192-metal": {NumCores: 192, MemoryGiB: 768, IsVirtual: false}, + "n1-standard-4": {NumCores: 4, MemoryGiB: 15, IsVirtual: true}, + "n2-standard-4": {NumCores: 4, MemoryGiB: 16, IsVirtual: true}, + "n2-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, + "n2d-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, + "g2-standard-8": {NumCores: 8, MemoryGiB: 32, IsVirtual: true}, + "ct4p-hightpu-4t": {NumCores: 240, MemoryGiB: 407, IsVirtual: true}, + "c3-standard-192-metal": { + NumCores: 192, + MemoryGiB: 768, + IsVirtual: false, + MaxPodCores: 144, + MaxPodMemoryGiB: 64, // More than this causes "Large hotplug" when using ACPI hotplugging. + }, } // TestCluster wraps clusters with their individual ClientSets so that helper methods can be called. @@ -730,17 +749,22 @@ func (t *TestCluster) applyCommonPodConfigurations(ctx context.Context, np *Node // Apply the runtime we've chosen, whether by override or autodetection. applyRuntime.ApplyPodSpec(podSpec) if applyRuntime.RequiresExplicitResourceLimits() { - const ( - overheadMargin = 0.2 - leftoverRatio = 1.0 - overheadMargin - ) - cores := int(float64(np.spec.NumCores) * leftoverRatio) - if cores < 1 { - cores = 1 + targetCores := np.spec.MaxPodCores + if targetCores == 0 { + targetCores = int(float64(np.spec.NumCores) * defaultMaxResourceUtilization) + } + if runtimeMaxCores := applyRuntime.MaxCores(); runtimeMaxCores != 0 && targetCores > runtimeMaxCores { + targetCores = runtimeMaxCores + } + if targetCores < 1 { + targetCores = 1 } - memMiB := int(float64(np.spec.MemoryGiB) * 1024 * leftoverRatio) - resCPU := resource.MustParse(fmt.Sprintf("%d", cores)) - resMem := resource.MustParse(fmt.Sprintf("%dMi", memMiB)) + targetMemoryMiB := np.spec.MaxPodMemoryGiB * 1024 + if targetMemoryMiB == 0 { + targetMemoryMiB = int(float64(np.spec.MemoryGiB*1024) * defaultMaxResourceUtilization) + } + resCPU := resource.MustParse(fmt.Sprintf("%d", targetCores)) + resMem := resource.MustParse(fmt.Sprintf("%dMi", targetMemoryMiB)) for _, containers := range [][]v13.Container{ podSpec.InitContainers, podSpec.Containers, @@ -965,6 +989,16 @@ func (t *TestCluster) ExecRequestInClientPod(ctx context.Context, service *v13.S return []byte(logs), nil } +// SupportsPersistentVolumes returns whether the cluster supports persistent volumes. +func (t *TestCluster) SupportsPersistentVolumes(ctx context.Context) (bool, error) { + clientNodePool, err := t.getNodePool(ctx, ClientNodepoolName) + if err != nil { + return false, fmt.Errorf("failed to get client nodepool: %w", err) + } + // Only virtual machines support persistent volumes currently. + return clientNodePool.spec.IsVirtual, nil +} + // CreatePersistentVolume creates a persistent volume. func (t *TestCluster) CreatePersistentVolume(ctx context.Context, volume *v13.PersistentVolumeClaim) (*v13.PersistentVolumeClaim, error) { if volume.GetObjectMeta().GetNamespace() == "" {