Adds Remote Write changes for Prometheus (#325)

* Adds Remote Write changes for Prometheus This commit adds changes to remote write Prometheus metrics to a AMP workspace. Signed-off-by: Ashish Ranjan <[email protected]> * Address comments of the PR #325 This commit addresses the comments of PR #325. Signed-off-by: Ashish Ranjan <[email protected]> Signed-off-by: Ashish Ranjan <[email protected]>
awslabs · Dec 5, 2022 · a21a7a3 · a21a7a3
1 parent cf70d5c
commit a21a7a3
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 7 deletions.
diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml
@@ -16,6 +16,8 @@ spec:
   - name: results-bucket
   - name: slack-hook
   - name: slack-message
+  - name: amp-workspace-id
+    default: ""
   tasks:
   - name: slack-notification
     params:
@@ -91,6 +93,8 @@ spec:
       value: $(params.results-bucket)
     - name: nodes
       value: $(params.desired-nodes)
+    - name: amp-workspace-id
+      value: '$(params.amp-workspace-id)'
     runAfter:
     - install-fluentbit-addon
     - create-cw-agent-addon

diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml
@@ -22,6 +22,8 @@ spec:
   - name: vpc-cfn-url
   - name: kubernetes-version
     default: "1.23"
+  - name: amp-workspace-id
+    default: ""
   tasks:
   - name: slack-notification
     params:
@@ -100,6 +102,8 @@ spec:
       value: $(params.cluster-name)
     - name: endpoint
       value: $(params.endpoint)
+    - name: amp-workspace-id
+      value: '$(params.amp-workspace-id)'
     runAfter:
     - install-fluentbit-addon
     taskRef:

diff --git a/tests/pipelines/eks/upstream-load.yaml b/tests/pipelines/eks/upstream-load.yaml
@@ -47,6 +47,9 @@ spec:
     default: "15"
   - name: results-bucket
     description: "Results bucket with path of s3 to upload results"
+  - name: amp-workspace-id
+    description: The AMP workspace ID where remote write needs to happen.
+    default: ""
   tasks:
   - name: create-eks-cluster
     taskRef:
@@ -97,6 +100,8 @@ spec:
         value: '$(params.results-bucket)'
       - name: nodes
         value: '$(params.desired-nodes)'
+      - name: amp-workspace-id
+        value: '$(params.amp-workspace-id)'
     workspaces:
       - name: source
         workspace: source

diff --git a/tests/tasks/generators/clusterloader/load.yaml b/tests/tasks/generators/clusterloader/load.yaml
@@ -35,11 +35,15 @@ spec:
     description: "aws eks enpoint to create clusters against"
   - name: cluster-name
     description: The name of the EKS cluster you want to spin.
+  - name: amp-workspace-id
+    description: The AMP workspace ID where remote write needs to happen.
+    default: ""
   results:
     - name: datapoint
       description: Stores the CL2 result that can be consumed by other tasks (e.g. cloudwatch) 
   workspaces:
   - name: source
+    mountPath: /src/k8s.io/
   - name: results
   steps:
   - name: git-clone      
@@ -67,11 +71,39 @@ spec:
       # we are not testing PVS at this point
       CL2_ENABLE_PVS: false
       PROMETHEUS_SCRAPE_KUBE_PROXY: true
+      PROMETHEUS_SCRAPE_APISERVER_ONLY: true
       ENABLE_SYSTEM_POD_METRICS: false
       NODE_MODE: master 
       EOL
       cat $(workspaces.source.path)/overrides.yaml
       cp $(workspaces.source.path)/overrides.yaml $(workspaces.results.path)/overrides.yaml
+      
+      # Enable Prometheus if the remote workspace id is provided
+      if [ -n "$(params.amp-workspace-id)" ]; then
+      # TODO: Currently pathing the prometheus manifests for remote write. Move this to upsteam going forward.
+      echo "volumeBindingMode: WaitForFirstConsumer" >> $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0ssd-storage-class.yaml
+      cat $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/0ssd-storage-class.yaml
+      cat << EOF >> $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
+        containers:
+          - name: aws-sigv4-proxy-sidecar
+            image: public.ecr.aws/aws-observability/aws-sigv4-proxy:1.0
+            args:
+              - --name
+              - aps
+              - --region
+              - $(params.region)
+              - --host
+              - aps-workspaces.$(params.region).amazonaws.com
+              - --port
+              - :8005
+            ports:
+              - name: aws-sigv4-proxy
+                containerPort: 8005
+        remoteWrite:
+          - url: http://localhost:8005/workspaces/$(params.amp-workspace-id)/api/v1/remote_write
+      EOF
+      cat $(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/prometheus-prometheus.yaml
+      fi
   - name: run-loadtest
     image: public.ecr.aws/kit/clusterloader2:0213bea
     onError: continue
@@ -81,9 +113,12 @@ spec:
       if [ -n "$(params.endpoint)" ]; then
         ENDPOINT_FLAG="--endpoint $(params.endpoint)"
       fi
+      if [ -n "$(params.amp-workspace-id)" ]; then
+        CL2_PROMETHEUS_FLAGS="--enable-prometheus-server=true --prometheus-storage-class-provisioner kubernetes.io/aws-ebs --prometheus-storage-class-volume-type gp2 --prometheus-manifest-path=$(workspaces.source.path)/perf-tests/clusterloader2/pkg/prometheus/manifests/"
+      fi
       aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
       cat $(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml
-      ENABLE_EXEC_SERVICE=false /clusterloader --kubeconfig=/root/.kube/config --testconfig=$(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml --testoverrides=$(workspaces.source.path)/overrides.yaml --nodes=$(params.nodes) --provider=eks --report-dir=$(workspaces.results.path) --alsologtostderr --v=2
+      ENABLE_EXEC_SERVICE=false /clusterloader --kubeconfig=/root/.kube/config --testconfig=$(workspaces.source.path)/perf-tests/clusterloader2/testing/load/config.yaml --testoverrides=$(workspaces.source.path)/overrides.yaml --nodes=$(params.nodes) --provider=eks --report-dir=$(workspaces.results.path) --alsologtostderr --v=2 $CL2_PROMETHEUS_FLAGS
       if [ $? -eq 0 ]; then
       echo "1" | tee $(results.datapoint.path)
       else

diff --git a/tests/tasks/setup/eks/awscli-cp-with-vpc.yaml b/tests/tasks/setup/eks/awscli-cp-with-vpc.yaml
@@ -24,6 +24,9 @@ spec:
     description: servicerole arn to be used for eks cluster to perform operations in customer account to setup cluster
   - name: vpc-stack-name
     description: The name of the VPC name you want to use for EKS cluster.
+  - name: aws-ebs-csi-driver-version
+    default: release-1.13
+    description: The release version for aws ebs csi driver.
   steps:
   - name: create-eks
     image: alpine/k8s:1.23.13
@@ -49,3 +52,5 @@ spec:
       aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.region)
       # enable PD on the cluster 
       kubectl set env daemonset/aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true
+      # install csi drivers.
+      kubectl apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=$(params.aws-ebs-csi-driver-version)"
diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml
@@ -49,21 +49,27 @@ spec:
       --query cluster.resourcesVpcConfig.subnetIds --output text \
       )
 
-      nodes=$(params.desired-nodes)
+      nodes=$(($(params.desired-nodes)-1))
       asgs=$((nodes/1000))
       asg_name=$(params.cluster-name)-nodes
       create_dp()
       {
         CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name)  --query 'nodegroups[?@==`'$asg_name-$1'`]' --output text)
-        #ToDo: paramterize instance-types
+        echo $3
+        EC2_INSTANCES=$3
+        echo $EC2_INSTANCES
+        if [ "$EC2_INSTANCES" == "" ]; then
+          # Defaulting to below instances if no other instances has been provided in the parameter.
+          EC2_INSTANCES="c5.large m5.large r5.large t3.large t3a.large c5a.large m5a.large r5a.large"
+        fi
         if [ "$CREATED_NODEGROUP" == "" ]; then
           #create node group
           aws eks $ENDPOINT_FLAG create-nodegroup \
           --cluster-name $(params.cluster-name) \
           --nodegroup-name $asg_name-$1 \
           --node-role $(params.host-cluster-node-role-arn) \
           --region $(params.region) \
-          --instance-types c5.large m5.large r5.large t3.large t3a.large c5a.large m5a.large r5a.large \
+          --instance-types $EC2_INSTANCES \
           --scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
           --subnets $NG_SUBNETS
         fi
@@ -77,15 +83,16 @@ spec:
       for i in $(seq 1 $asgs)
       do
         #max number of nodes MNG allows per ASG
-        create_dp $i 1000
+        create_dp $i 1000 
       done
-      remaining_nodes=$((nodes%1000))
+      remaining_nodes=$(((nodes)%1000))
       if [[ $remaining_nodes -gt 0 ]]
       then
         echo "The remaining_nodes var is greater than 0."
         create_dp 0 $remaining_nodes
       fi
-      
+      # Creating an extra asg with 1 large node to ensure prometheus server has a scheduling space.
+      create_dp $(($asgs + 1)) 1 m5.4xlarge
   - name: validate-nodes
     image: alpine/k8s:1.22.6
     script: |