diff --git a/api/v1beta3/tortoise_types.go b/api/v1beta3/tortoise_types.go index 55b4e7b9..5fa4b0d6 100644 --- a/api/v1beta3/tortoise_types.go +++ b/api/v1beta3/tortoise_types.go @@ -62,37 +62,26 @@ type TortoiseSpec struct { // "NoDelete" is the default value. // +optional DeletionPolicy DeletionPolicy `json:"deletionPolicy,omitempty" protobuf:"bytes,4,opt,name=deletionPolicy"` - // AutoscalingPolicy is an optional field to specify how each resource in each container is scaled. + // AutoscalingPolicy is an optional field for specifying the scaling approach for each resource within each container. // - // You basically have two options to configure how each resource in each container is scaled - - // 1. you let tortoise manage which resource is scaled with which autoscaling policy. - // 2. you manage which resource is scaled with which autoscaling policy by yourself. + // There are two primary options for configuring resource scaling within containers: + // 1. Allow Tortoise to automatically determine the appropriate autoscaling policy for each resource. + // 2. Manually define the autoscaling policy for each resource. // - // For the first option, you just have to leave this field empty. - // Then, when tortoise is created, tortoise will configure the AutoscalingPolicy for each resource in each container based on the following rules: - // - If .spec.TargetRefs.HorizontalPodAutoscalerName is empty, the initial policy being used is "Horizontal" for cpu, and "Vertical" for memory. - // - If .spec.TargetRefs.HorizontalPodAutoscalerName is not empty, tortoise sets "Horizontal" to resources managed by the attached HPA, - // and "Vertical" to resources not managed by the attached HPA. - // Also, when a new container is added to the workload, - // tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container. - // ("Horizontal" for cpu, and "Vertical" for memory) + // For the first option, simply leave this field unset. In this case, Tortoise will adjust the autoscaling policies using the following logic: + // - If .spec.TargetRefs.HorizontalPodAutoscalerName is not provided, the policies default to "Horizontal" for CPU and "Vertical" for memory across all containers. + // - If .spec.TargetRefs.HorizontalPodAutoscalerName is specified, resources governed by the referenced Horizontal Pod Autoscaler will use a "Horizontal" policy, + // while those not managed by the HPA will use a "Vertical" policy. + // Note that Tortoise supports only the ContainerResource metric type for HPAs; other metric types will be disregarded. + // Additionally, if a ContainerResource metric is later added to an HPA associated with Tortoise, + // Tortoise will automatically update relevant resources to utilize a "Horizontal" policy. // - // For the second option, you have to specify the AutoscalingPolicy for each resource in each container in this field. - // If you specify the AutoscalingPolicy for some containers, but not for all, - // tortoise will set the default value "Off" to the all resources in the containers not specified. - // Note that when a new container is added to the workload, you have to update the AutoscalingPolicy field by yourself, - // otherwise, tortoise will never scale the resources in the new container. - // - // Those two options are created for different use cases: - // - Basically, you should use the first option that keep configuring the AutoscalingPolicy for each resource in each container automatically. - // - If you have a special reason that you want to scale, for example, memory with horizontal scaling, - // you should use the second option that provides you the flexibility to configure the AutoscalingPolicy for each resource in each container. - // - // This field is mutable - you can even change this field from non-empty to empty. - // In that case, tortoise will just keep the way each container is scaled, - // but, you can obtain the merit of the first option - when a new container is added to the workload, - // tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container automatically. + // With the second option, you must manually specify the AutoscalingPolicy for the resources of each container within this field. + // If policies are defined for some but not all containers or resources, Tortoise will assign a default "Off" policy to unspecified resources. + // Be aware that when new containers are introduced to the workload, the AutoscalingPolicy configuration must be manually updated, + // as Tortoise will default to an "Off" policy for resources within the new container, preventing scaling. // + // The AutoscalingPolicy field is mutable; you can modify it at any time, whether from an empty state to populated or vice versa. // +optional AutoscalingPolicy []ContainerAutoscalingPolicy `json:"autoscalingPolicy,omitempty" protobuf:"bytes,5,opt,name=autoscalingPolicy"` } diff --git a/config/crd/bases/autoscaling.mercari.com_tortoises.yaml b/config/crd/bases/autoscaling.mercari.com_tortoises.yaml index 48854ac3..106cdc8d 100644 --- a/config/crd/bases/autoscaling.mercari.com_tortoises.yaml +++ b/config/crd/bases/autoscaling.mercari.com_tortoises.yaml @@ -565,42 +565,33 @@ spec: description: TortoiseSpec defines the desired state of Tortoise properties: autoscalingPolicy: - description: "AutoscalingPolicy is an optional field to specify how - each resource in each container is scaled. \n You basically have - two options to configure how each resource in each container is - scaled - 1. you let tortoise manage which resource is scaled with - which autoscaling policy. 2. you manage which resource is scaled - with which autoscaling policy by yourself. \n For the first option, - you just have to leave this field empty. Then, when tortoise is - created, tortoise will configure the AutoscalingPolicy for each - resource in each container based on the following rules: - If .spec.TargetRefs.HorizontalPodAutoscalerName - is empty, the initial policy being used is \"Horizontal\" for cpu, - and \"Vertical\" for memory. - If .spec.TargetRefs.HorizontalPodAutoscalerName - is not empty, tortoise sets \"Horizontal\" to resources managed - by the attached HPA, and \"Vertical\" to resources not managed by - the attached HPA. Also, when a new container is added to the workload, - tortoise will notice it and configure the AutoscalingPolicy for - each resource in the new container. (\"Horizontal\" for cpu, and - \"Vertical\" for memory) \n For the second option, you have to specify - the AutoscalingPolicy for each resource in each container in this - field. If you specify the AutoscalingPolicy for some containers, - but not for all, tortoise will set the default value \"Off\" to - the all resources in the containers not specified. Note that when - a new container is added to the workload, you have to update the - AutoscalingPolicy field by yourself, otherwise, tortoise will never - scale the resources in the new container. \n Those two options are - created for different use cases: - Basically, you should use the - first option that keep configuring the AutoscalingPolicy for each - resource in each container automatically. - If you have a special - reason that you want to scale, for example, memory with horizontal - scaling, you should use the second option that provides you the - flexibility to configure the AutoscalingPolicy for each resource - in each container. \n This field is mutable - you can even change - this field from non-empty to empty. In that case, tortoise will - just keep the way each container is scaled, but, you can obtain - the merit of the first option - when a new container is added to - the workload, tortoise will notice it and configure the AutoscalingPolicy - for each resource in the new container automatically." + description: "AutoscalingPolicy is an optional field for specifying + the scaling approach for each resource within each container. \n + There are two primary options for configuring resource scaling within + containers: 1. Allow Tortoise to automatically determine the appropriate + autoscaling policy for each resource. 2. Manually define the autoscaling + policy for each resource. \n For the first option, simply leave + this field unset. In this case, Tortoise will adjust the autoscaling + policies using the following logic: - If .spec.TargetRefs.HorizontalPodAutoscalerName + is not provided, the policies default to \"Horizontal\" for CPU + and \"Vertical\" for memory across all containers. - If .spec.TargetRefs.HorizontalPodAutoscalerName + is specified, resources governed by the referenced Horizontal Pod + Autoscaler will use a \"Horizontal\" policy, while those not managed + by the HPA will use a \"Vertical\" policy. Note that Tortoise supports + only the ContainerResource metric type for HPAs; other metric types + will be disregarded. Additionally, if a ContainerResource metric + is later added to an HPA associated with Tortoise, Tortoise will + automatically update relevant resources to utilize a \"Horizontal\" + policy. \n With the second option, you must manually specify the + AutoscalingPolicy for the resources of each container within this + field. If policies are defined for some but not all containers or + resources, Tortoise will assign a default \"Off\" policy to unspecified + resources. Be aware that when new containers are introduced to the + workload, the AutoscalingPolicy configuration must be manually updated, + as Tortoise will default to an \"Off\" policy for resources within + the new container, preventing scaling. \n The AutoscalingPolicy + field is mutable; you can modify it at any time, whether from an + empty state to populated or vice versa." items: properties: containerName: diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml new file mode 100644 index 00000000..527dabb9 --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml @@ -0,0 +1,45 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + tortoises.autoscaling.mercari.com/tortoise-name: mercari + name: tortoise-hpa-mercari + namespace: default +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 20 + metrics: + - external: + metric: + name: datadogmetric@hoge:hoge + target: + type: Value + value: 90 + type: External + - containerResource: + container: app + name: memory # changed + target: + averageUtilization: 75 + type: Utilization + type: ContainerResource + minReplicas: 5 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml new file mode 100644 index 00000000..791f4cc0 --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml @@ -0,0 +1,77 @@ +metadata: + name: mercari + namespace: default +spec: + targetRefs: + horizontalPodAutoscalerName: tortoise-hpa-mercari + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + autoscalingPolicy: + - policy: + cpu: Vertical + memory: Horizontal + containerName: app + conditions: + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:01:24Z" + value: 20 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:01:24Z" + value: 5 + targetUtilizations: + - containerName: app + targetUtilization: + memory: 75 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "3" + memory: 4Gi + containerName: app + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + verticalPodAutoscalers: + - name: tortoise-updater-mercari + role: Updater + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working + containerResourcePhases: + - containerName: "app" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working \ No newline at end of file diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/vpa-Monitor.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/vpa-Monitor.yaml similarity index 100% rename from controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/vpa-Monitor.yaml rename to controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/vpa-Monitor.yaml diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/vpa-Updater.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/vpa-Updater.yaml new file mode 100644 index 00000000..91db2a2b --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/vpa-Updater.yaml @@ -0,0 +1,34 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + tortoises.autoscaling.mercari.com/tortoise-name: mercari + name: tortoise-updater-mercari + namespace: default +spec: + recommenders: + - name: tortoise-controller + autoscalingPolicy: + containerPolicies: + - containerName: app + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: Auto +status: + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 4Gi + target: + cpu: "3" + memory: 4Gi + uncappedTarget: + cpu: "3" + memory: 4Gi + upperBound: + cpu: "3" + memory: 4Gi diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/deployment.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/deployment.yaml new file mode 100644 index 00000000..34693daa --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/deployment.yaml @@ -0,0 +1,23 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "4" + memory: 4Gi +status: + replicas: 10 diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml new file mode 100644 index 00000000..b4e06e9f --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml @@ -0,0 +1,45 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + tortoises.autoscaling.mercari.com/tortoise-name: mercari + name: tortoise-hpa-mercari + namespace: default +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 100 + metrics: + - external: + metric: + name: datadogmetric@hoge:hoge + target: + type: Value + value: 90 + type: External + - containerResource: + container: app + name: memory + target: + averageUtilization: 50 + type: Utilization + type: ContainerResource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + currentMetrics: null + desiredReplicas: 0 diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/tortoise.yaml new file mode 100644 index 00000000..b7365dda --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/tortoise.yaml @@ -0,0 +1,68 @@ +metadata: + name: mercari + namespace: default +spec: + targetRefs: + horizontalPodAutoscalerName: tortoise-hpa-mercari + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + autoscalingPolicy: + - policy: + cpu: Horizontal + memory: Vertical + containerName: app + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "0" + updatedAt: null + memory: + quantity: "0" + updatedAt: null + recommendation: + cpu: + quantity: "0" + updatedAt: null + memory: + quantity: "0" + updatedAt: null + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:01:24Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:01:24Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 50 + vertical: + containerResourceRecommendation: null + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + verticalPodAutoscalers: + - name: tortoise-updater-mercari + role: Updater + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working + containerResourcePhases: + - containerName: "app" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working \ No newline at end of file diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/vpa-Monitor.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/vpa-Monitor.yaml new file mode 100644 index 00000000..bfeaf1a1 --- /dev/null +++ b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/vpa-Monitor.yaml @@ -0,0 +1,33 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + tortoises.autoscaling.mercari.com/tortoise-name: mercari + name: tortoise-monitor-mercari + namespace: default +spec: + autoscalingPolicy: + containerPolicies: + - containerName: app + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/vpa-Updater.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/vpa-Updater.yaml similarity index 100% rename from controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/vpa-Updater.yaml rename to controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/vpa-Updater.yaml diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml index b76a3d33..2948735a 100644 --- a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml +++ b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml @@ -19,7 +19,7 @@ spec: value: 100 selectPolicy: Max stabilizationWindowSeconds: 0 - maxReplicas: 20 + maxReplicas: 100 metrics: - containerResource: container: app @@ -28,7 +28,7 @@ spec: averageUtilization: 50 type: Utilization type: ContainerResource - minReplicas: 5 + minReplicas: 1 scaleTargetRef: apiVersion: apps/v1 kind: Deployment diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml index b52e99d2..21d8d3ef 100644 --- a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml +++ b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml @@ -3,6 +3,7 @@ metadata: namespace: default spec: targetRefs: + horizontalPodAutoscalerName: tortoise-hpa-mercari scaleTargetRef: apiVersion: apps/v1 kind: Deployment @@ -14,63 +15,58 @@ status: memory: Vertical containerName: app conditions: - tortoiseConditions: - - message: "HPA target utilization is updated" - reason: HPATargetUtilizationUpdated - status: "True" - type: HPATargetUtilizationUpdated containerRecommendationFromVPA: - containerName: app maxRecommendation: cpu: - quantity: "3" + quantity: 0 updatedAt: null memory: - quantity: 3Gi + quantity: 0 updatedAt: null recommendation: cpu: - quantity: "3" + quantity: 0 updatedAt: null memory: - quantity: 3Gi + quantity: 0 updatedAt: null recommendations: horizontal: maxReplicas: - from: 0 - timezone: Local + timezone: "Asia/Tokyo" to: 24 updatedAt: "2023-10-06T01:01:24Z" - value: 20 + value: 0 minReplicas: - from: 0 - timezone: Local + timezone: "Asia/Tokyo" to: 24 updatedAt: "2023-10-06T01:01:24Z" - value: 5 - targetUtilizations: - - containerName: app - targetUtilization: - cpu: 75 + value: 0 vertical: containerResourceRecommendation: - RecommendedResource: cpu: "4" - memory: 3Gi + memory: 4Gi containerName: app targets: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app horizontalPodAutoscaler: tortoise-hpa-mercari verticalPodAutoscalers: - - name: tortoise-updater-mercari - role: Updater - name: tortoise-monitor-mercari role: Monitor - tortoisePhase: GatheringData + - name: tortoise-updater-mercari + role: Updater + tortoisePhase: Initializing containerResourcePhases: - containerName: "app" resourcePhases: cpu: phase: GatheringData memory: - phase: GatheringData \ No newline at end of file + phase: GatheringData \ No newline at end of file diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/vpa-Monitor.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/vpa-Monitor.yaml index bfeaf1a1..13a014e3 100644 --- a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/vpa-Monitor.yaml +++ b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/vpa-Monitor.yaml @@ -13,21 +13,4 @@ spec: kind: Deployment name: mercari-app updatePolicy: - updateMode: "Off" -status: - conditions: - - lastTransitionTime: null - status: "True" - type: RecommendationProvided - recommendation: - containerRecommendations: - - containerName: app - lowerBound: - cpu: "3" - memory: 3Gi - target: - cpu: "3" - memory: 3Gi - upperBound: - cpu: "5" - memory: 5Gi + updateMode: "Off" \ No newline at end of file diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/tortoise.yaml index 511609b7..2b5ce95d 100644 --- a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/tortoise.yaml +++ b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/tortoise.yaml @@ -1,71 +1,12 @@ metadata: name: mercari namespace: default + annotations: + "skip-status-update": "true" spec: targetRefs: + horizontalPodAutoscalerName: tortoise-hpa-mercari # Autoscaling policy will be generated from HPA scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: mercari-app -status: - autoscalingPolicy: - - policy: - cpu: Horizontal - memory: Vertical - containerName: app - conditions: - containerRecommendationFromVPA: - - containerName: app - maxRecommendation: - cpu: - quantity: "3" - updatedAt: null - memory: - quantity: 3Gi - updatedAt: null - recommendation: - cpu: - quantity: "3" - updatedAt: null - memory: - quantity: 3Gi - updatedAt: null - recommendations: - horizontal: - maxReplicas: - - from: 0 - timezone: Local - to: 24 - updatedAt: "2023-10-06T01:01:24Z" - value: 20 - minReplicas: - - from: 0 - timezone: Local - to: 24 - updatedAt: "2023-10-06T01:01:24Z" - value: 5 - targetUtilizations: - - containerName: app - targetUtilization: - cpu: 75 - vertical: - containerResourceRecommendation: - - RecommendedResource: - cpu: "4" - memory: 3Gi - containerName: app - targets: - horizontalPodAutoscaler: tortoise-hpa-mercari - verticalPodAutoscalers: - - name: tortoise-updater-mercari - role: Updater - - name: tortoise-monitor-mercari - role: Monitor - tortoisePhase: GatheringData - containerResourcePhases: - - containerName: "app" - resourcePhases: - cpu: - phase: GatheringData - memory: - phase: GatheringData \ No newline at end of file + name: mercari-app \ No newline at end of file diff --git a/controllers/tortoise_controller_test.go b/controllers/tortoise_controller_test.go index d4b0d987..7b21baf9 100644 --- a/controllers/tortoise_controller_test.go +++ b/controllers/tortoise_controller_test.go @@ -122,6 +122,9 @@ func createTortoiseWithStatus(ctx context.Context, k8sClient client.Client, tort err = k8sClient.Get(ctx, client.ObjectKey{Namespace: tortoise.Namespace, Name: tortoise.Name}, v) Expect(err).NotTo(HaveOccurred()) + if tortoise.Annotations["skip-status-update"] == "true" { + return + } v.Status = tortoise.Status err = k8sClient.Status().Update(ctx, v) Expect(err).NotTo(HaveOccurred()) @@ -135,8 +138,12 @@ func initializeResourcesFromFiles(ctx context.Context, k8sClient client.Client, } createDeploymentWithStatus(ctx, k8sClient, resource.deployment) - createVPAWithStatus(ctx, k8sClient, resource.vpa[v1beta3.VerticalPodAutoscalerRoleUpdater]) - createVPAWithStatus(ctx, k8sClient, resource.vpa[v1beta3.VerticalPodAutoscalerRoleMonitor]) + if resource.vpa[v1beta3.VerticalPodAutoscalerRoleUpdater] != nil { + createVPAWithStatus(ctx, k8sClient, resource.vpa[v1beta3.VerticalPodAutoscalerRoleUpdater]) + } + if resource.vpa[v1beta3.VerticalPodAutoscalerRoleMonitor] != nil { + createVPAWithStatus(ctx, k8sClient, resource.vpa[v1beta3.VerticalPodAutoscalerRoleMonitor]) + } createTortoiseWithStatus(ctx, k8sClient, resource.tortoise) return resource @@ -151,7 +158,7 @@ func startController(ctx context.Context) func() { Expect(err).ShouldNot(HaveOccurred()) // We only reconcile once. - tortoiseService, err := tortoise.New(mgr.GetClient(), record.NewFakeRecorder(10), 1, "Asia/Tokyo", 1000*time.Minute, "weekly") + tortoiseService, err := tortoise.New(mgr.GetClient(), record.NewFakeRecorder(10), 24, "Asia/Tokyo", 1000*time.Minute, "daily") Expect(err).ShouldNot(HaveOccurred()) cli, err := vpa.New(mgr.GetConfig(), record.NewFakeRecorder(10)) Expect(err).ShouldNot(HaveOccurred()) @@ -316,12 +323,18 @@ var _ = Describe("Test TortoiseController", func() { checkWithWantedResources(path) cleanUp() }) + It("TortoisePhaseInitializing", func() { + runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-initializing")) + }) It("TortoisePhaseWorking (GatheringData is just finished)", func() { runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-gathering-data-finished")) }) It("TortoisePhaseWorking (dryrun)", func() { runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-dryrun")) }) + It("TortoisePhaseWorking and HPA changed", func() { + runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-hpa-changed")) + }) It("TortoisePhaseEmergency", func() { runTest(filepath.Join("testdata", "reconcile-for-the-single-container-pod-emergency")) }) diff --git a/manifests/crd/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml b/manifests/crd/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml index 34f200e2..1a86888d 100644 --- a/manifests/crd/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml +++ b/manifests/crd/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml @@ -444,7 +444,7 @@ spec: description: TortoiseSpec defines the desired state of Tortoise properties: autoscalingPolicy: - description: "AutoscalingPolicy is an optional field to specify how each resource in each container is scaled. \n You basically have two options to configure how each resource in each container is scaled - 1. you let tortoise manage which resource is scaled with which autoscaling policy. 2. you manage which resource is scaled with which autoscaling policy by yourself. \n For the first option, you just have to leave this field empty. Then, when tortoise is created, tortoise will configure the AutoscalingPolicy for each resource in each container based on the following rules: - If .spec.TargetRefs.HorizontalPodAutoscalerName is empty, the initial policy being used is \"Horizontal\" for cpu, and \"Vertical\" for memory. - If .spec.TargetRefs.HorizontalPodAutoscalerName is not empty, tortoise sets \"Horizontal\" to resources managed by the attached HPA, and \"Vertical\" to resources not managed by the attached HPA. Also, when a new container is added to the workload, tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container. (\"Horizontal\" for cpu, and \"Vertical\" for memory) \n For the second option, you have to specify the AutoscalingPolicy for each resource in each container in this field. If you specify the AutoscalingPolicy for some containers, but not for all, tortoise will set the default value \"Off\" to the all resources in the containers not specified. Note that when a new container is added to the workload, you have to update the AutoscalingPolicy field by yourself, otherwise, tortoise will never scale the resources in the new container. \n Those two options are created for different use cases: - Basically, you should use the first option that keep configuring the AutoscalingPolicy for each resource in each container automatically. - If you have a special reason that you want to scale, for example, memory with horizontal scaling, you should use the second option that provides you the flexibility to configure the AutoscalingPolicy for each resource in each container. \n This field is mutable - you can even change this field from non-empty to empty. In that case, tortoise will just keep the way each container is scaled, but, you can obtain the merit of the first option - when a new container is added to the workload, tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container automatically." + description: "AutoscalingPolicy is an optional field for specifying the scaling approach for each resource within each container. \n There are two primary options for configuring resource scaling within containers: 1. Allow Tortoise to automatically determine the appropriate autoscaling policy for each resource. 2. Manually define the autoscaling policy for each resource. \n For the first option, simply leave this field unset. In this case, Tortoise will adjust the autoscaling policies using the following logic: - If .spec.TargetRefs.HorizontalPodAutoscalerName is not provided, the policies default to \"Horizontal\" for CPU and \"Vertical\" for memory across all containers. - If .spec.TargetRefs.HorizontalPodAutoscalerName is specified, resources governed by the referenced Horizontal Pod Autoscaler will use a \"Horizontal\" policy, while those not managed by the HPA will use a \"Vertical\" policy. Note that Tortoise supports only the ContainerResource metric type for HPAs; other metric types will be disregarded. Additionally, if a ContainerResource metric is later added to an HPA associated with Tortoise, Tortoise will automatically update relevant resources to utilize a \"Horizontal\" policy. \n With the second option, you must manually specify the AutoscalingPolicy for the resources of each container within this field. If policies are defined for some but not all containers or resources, Tortoise will assign a default \"Off\" policy to unspecified resources. Be aware that when new containers are introduced to the workload, the AutoscalingPolicy configuration must be manually updated, as Tortoise will default to an \"Off\" policy for resources within the new container, preventing scaling. \n The AutoscalingPolicy field is mutable; you can modify it at any time, whether from an empty state to populated or vice versa." items: properties: containerName: diff --git a/manifests/default/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml b/manifests/default/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml index 535d688b..d9e3a893 100644 --- a/manifests/default/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml +++ b/manifests/default/apiextensions.k8s.io_v1_customresourcedefinition_tortoises.autoscaling.mercari.com.yaml @@ -444,7 +444,7 @@ spec: description: TortoiseSpec defines the desired state of Tortoise properties: autoscalingPolicy: - description: "AutoscalingPolicy is an optional field to specify how each resource in each container is scaled. \n You basically have two options to configure how each resource in each container is scaled - 1. you let tortoise manage which resource is scaled with which autoscaling policy. 2. you manage which resource is scaled with which autoscaling policy by yourself. \n For the first option, you just have to leave this field empty. Then, when tortoise is created, tortoise will configure the AutoscalingPolicy for each resource in each container based on the following rules: - If .spec.TargetRefs.HorizontalPodAutoscalerName is empty, the initial policy being used is \"Horizontal\" for cpu, and \"Vertical\" for memory. - If .spec.TargetRefs.HorizontalPodAutoscalerName is not empty, tortoise sets \"Horizontal\" to resources managed by the attached HPA, and \"Vertical\" to resources not managed by the attached HPA. Also, when a new container is added to the workload, tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container. (\"Horizontal\" for cpu, and \"Vertical\" for memory) \n For the second option, you have to specify the AutoscalingPolicy for each resource in each container in this field. If you specify the AutoscalingPolicy for some containers, but not for all, tortoise will set the default value \"Off\" to the all resources in the containers not specified. Note that when a new container is added to the workload, you have to update the AutoscalingPolicy field by yourself, otherwise, tortoise will never scale the resources in the new container. \n Those two options are created for different use cases: - Basically, you should use the first option that keep configuring the AutoscalingPolicy for each resource in each container automatically. - If you have a special reason that you want to scale, for example, memory with horizontal scaling, you should use the second option that provides you the flexibility to configure the AutoscalingPolicy for each resource in each container. \n This field is mutable - you can even change this field from non-empty to empty. In that case, tortoise will just keep the way each container is scaled, but, you can obtain the merit of the first option - when a new container is added to the workload, tortoise will notice it and configure the AutoscalingPolicy for each resource in the new container automatically." + description: "AutoscalingPolicy is an optional field for specifying the scaling approach for each resource within each container. \n There are two primary options for configuring resource scaling within containers: 1. Allow Tortoise to automatically determine the appropriate autoscaling policy for each resource. 2. Manually define the autoscaling policy for each resource. \n For the first option, simply leave this field unset. In this case, Tortoise will adjust the autoscaling policies using the following logic: - If .spec.TargetRefs.HorizontalPodAutoscalerName is not provided, the policies default to \"Horizontal\" for CPU and \"Vertical\" for memory across all containers. - If .spec.TargetRefs.HorizontalPodAutoscalerName is specified, resources governed by the referenced Horizontal Pod Autoscaler will use a \"Horizontal\" policy, while those not managed by the HPA will use a \"Vertical\" policy. Note that Tortoise supports only the ContainerResource metric type for HPAs; other metric types will be disregarded. Additionally, if a ContainerResource metric is later added to an HPA associated with Tortoise, Tortoise will automatically update relevant resources to utilize a \"Horizontal\" policy. \n With the second option, you must manually specify the AutoscalingPolicy for the resources of each container within this field. If policies are defined for some but not all containers or resources, Tortoise will assign a default \"Off\" policy to unspecified resources. Be aware that when new containers are introduced to the workload, the AutoscalingPolicy configuration must be manually updated, as Tortoise will default to an \"Off\" policy for resources within the new container, preventing scaling. \n The AutoscalingPolicy field is mutable; you can modify it at any time, whether from an empty state to populated or vice versa." items: properties: containerName: diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index 65fff422..820338a3 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -141,9 +141,9 @@ type resourceNameAndContainerName struct { containerName string } -// addHPAMetricsFromTortoiseAutoscalingPolicy adds metrics to the HPA based on the autoscaling policy in the tortoise. +// syncHPAMetricsWithTortoiseAutoscalingPolicy adds metrics to the HPA based on the autoscaling policy in the tortoise. // Note that it doesn't update the HPA in kube-apiserver, you have to do that after this function. -func (c *Service) addHPAMetricsFromTortoiseAutoscalingPolicy(ctx context.Context, tortoise *autoscalingv1beta3.Tortoise, currenthpa *v2.HorizontalPodAutoscaler, now time.Time) (*v2.HorizontalPodAutoscaler, *autoscalingv1beta3.Tortoise, bool) { +func (c *Service) syncHPAMetricsWithTortoiseAutoscalingPolicy(ctx context.Context, tortoise *autoscalingv1beta3.Tortoise, currenthpa *v2.HorizontalPodAutoscaler, now time.Time) (*v2.HorizontalPodAutoscaler, *autoscalingv1beta3.Tortoise, bool) { hpaEdited := false policies := sets.New[string]() @@ -217,9 +217,16 @@ func (c *Service) addHPAMetricsFromTortoiseAutoscalingPolicy(ctx context.Context // remove metrics newMetrics := []v2.MetricSpec{} for _, m := range currenthpa.Spec.Metrics { + if m.Type == v2.ResourceMetricSourceType { + // resource metrics should be removed. + continue + } if m.Type != v2.ContainerResourceMetricSourceType { + // We keep container resource metrics. + newMetrics = append(newMetrics, m) continue } + if !needToRemoveFromHPA.Has(resourceNameAndContainerName{m.ContainerResource.Name, m.ContainerResource.Container}) { newMetrics = append(newMetrics, m) hpaEdited = true @@ -281,7 +288,7 @@ func (c *Service) CreateHPA(ctx context.Context, tortoise *autoscalingv1beta3.To }, } - hpa, tortoise, _ = c.addHPAMetricsFromTortoiseAutoscalingPolicy(ctx, tortoise, hpa, now) + hpa, tortoise, _ = c.syncHPAMetricsWithTortoiseAutoscalingPolicy(ctx, tortoise, hpa, now) tortoise.Status.Targets.HorizontalPodAutoscaler = hpa.Name @@ -480,7 +487,7 @@ func (c *Service) UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx context.Context var newhpa *v2.HorizontalPodAutoscaler var isHpaEdited bool - newhpa, tortoise, isHpaEdited = c.addHPAMetricsFromTortoiseAutoscalingPolicy(ctx, tortoise, hpa, now) + newhpa, tortoise, isHpaEdited = c.syncHPAMetricsWithTortoiseAutoscalingPolicy(ctx, tortoise, hpa, now) if !isHpaEdited { // User didn't change anything. return tortoise, nil diff --git a/pkg/hpa/service_test.go b/pkg/hpa/service_test.go index 38eac512..9135d537 100644 --- a/pkg/hpa/service_test.go +++ b/pkg/hpa/service_test.go @@ -2381,6 +2381,727 @@ func TestService_UpdateHPASpecFromTortoiseAutoscalingPolicy(t *testing.T) { }, afterHPA: nil, }, + { + name: "remove metrics from hpa, but not remove the external metrics", + args: args{ + tortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeVertical, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + replicaNum: 4, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + wantTortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeVertical, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + afterHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + }, + { + name: "add metrics to existing hpa (with external metrics)", + args: args{ + tortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + replicaNum: 4, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + wantTortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseGatheringData, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + afterHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + }, + { + name: "Resource type metric is removed", + args: args{ + tortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeVertical, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + replicaNum: 4, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ResourceMetricSourceType, + Resource: &v2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + wantTortoise: &autoscalingv1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tortoise", + Namespace: "default", + }, + Spec: autoscalingv1beta3.TortoiseSpec{ + TargetRefs: autoscalingv1beta3.TargetRefs{ + HorizontalPodAutoscalerName: pointer.String("existing-hpa"), + ScaleTargetRef: autoscalingv1beta3.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + }, + }, + }, + Status: autoscalingv1beta3.TortoiseStatus{ + AutoscalingPolicy: []autoscalingv1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeVertical, + }, + }, + { + ContainerName: "istio-proxy", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeVertical, + v1.ResourceCPU: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + ContainerResourcePhases: []autoscalingv1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + { + ContainerName: "istio-proxy", + ResourcePhases: map[v1.ResourceName]autoscalingv1beta3.ResourcePhase{ + v1.ResourceCPU: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + v1.ResourceMemory: { + Phase: autoscalingv1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: autoscalingv1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + }, + }, + afterHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + Namespace: "default", + Annotations: map[string]string{ + annotation.ManagedByTortoiseAnnotation: "true", + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(1), + MaxReplicas: 2, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: pointer.Int32(50), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ExternalMetricSourceType, + External: &v2.ExternalMetricSource{ + Metric: v2.MetricIdentifier{ + Name: "external-metric", + }, + }, + }, + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {