Added CLI options

Signed-off-by: Vishal Biyani <[email protected]>

scripts/cluster-setup/README.md

@@ -0,0 +1,10 @@
+
+echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-1-15" | vegeta attack -duration=300s -timeout=3600s -rate=1200/m -max-workers=2000> hello-1-15.txt
+
+echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-2-45" | vegeta attack -duration=300s -timeout=3600s -rate=700/m -max-workers=2000> hello-2-45.txt
+
+echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-3-75" | vegeta attack -duration=300s -timeout=3600s -rate=700/m -max-workers=2000> hello-3-75.txt
+
+echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-4-90" | vegeta attack -duration=300s -timeout=3600s -rate=500/m -max-workers=2000> hello-4-90.txt
+
+echo "GET http://a72c0989896a24498b58842b22da42a6-2126244409.ap-south-1.elb.amazonaws.com/fission-function/hello-5-120" | vegeta attack -duration=300s -timeout=3600s -rate=500/m -max-workers=2000> hello-5-120.txt

scripts/cluster-setup/cluster-as.yaml

@@ -1,170 +1,169 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+  name: cluster-autoscaler
+  namespace: kube-system
 ---
-    apiVersion: v1
-    kind: ServiceAccount
-    metadata:
-      labels:
-        k8s-addon: cluster-autoscaler.addons.k8s.io
-        k8s-app: cluster-autoscaler
-      name: cluster-autoscaler
-      namespace: kube-system
-    ---
-    apiVersion: rbac.authorization.k8s.io/v1
-    kind: ClusterRole
-    metadata:
-      name: cluster-autoscaler
-      labels:
-        k8s-addon: cluster-autoscaler.addons.k8s.io
-        k8s-app: cluster-autoscaler
-    rules:
-      - apiGroups: [""]
-        resources: ["events", "endpoints"]
-        verbs: ["create", "patch"]
-      - apiGroups: [""]
-        resources: ["pods/eviction"]
-        verbs: ["create"]
-      - apiGroups: [""]
-        resources: ["pods/status"]
-        verbs: ["update"]
-      - apiGroups: [""]
-        resources: ["endpoints"]
-        resourceNames: ["cluster-autoscaler"]
-        verbs: ["get", "update"]
-      - apiGroups: [""]
-        resources: ["nodes"]
-        verbs: ["watch", "list", "get", "update"]
-      - apiGroups: [""]
-        resources:
-          - "pods"
-          - "services"
-          - "replicationcontrollers"
-          - "persistentvolumeclaims"
-          - "persistentvolumes"
-        verbs: ["watch", "list", "get"]
-      - apiGroups: ["extensions"]
-        resources: ["replicasets", "daemonsets"]
-        verbs: ["watch", "list", "get"]
-      - apiGroups: ["policy"]
-        resources: ["poddisruptionbudgets"]
-        verbs: ["watch", "list"]
-      - apiGroups: ["apps"]
-        resources: ["statefulsets", "replicasets", "daemonsets"]
-        verbs: ["watch", "list", "get"]
-      - apiGroups: ["storage.k8s.io"]
-        resources: ["storageclasses", "csinodes"]
-        verbs: ["watch", "list", "get"]
-      - apiGroups: ["batch", "extensions"]
-        resources: ["jobs"]
-        verbs: ["get", "list", "watch", "patch"]
-      - apiGroups: ["coordination.k8s.io"]
-        resources: ["leases"]
-        verbs: ["create"]
-      - apiGroups: ["coordination.k8s.io"]
-        resourceNames: ["cluster-autoscaler"]
-        resources: ["leases"]
-        verbs: ["get", "update"]
-    ---
-    apiVersion: rbac.authorization.k8s.io/v1
-    kind: Role
-    metadata:
-      name: cluster-autoscaler
-      namespace: kube-system
-      labels:
-        k8s-addon: cluster-autoscaler.addons.k8s.io
-        k8s-app: cluster-autoscaler
-    rules:
-      - apiGroups: [""]
-        resources: ["configmaps"]
-        verbs: ["create","list","watch"]
-      - apiGroups: [""]
-        resources: ["configmaps"]
-        resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
-        verbs: ["delete", "get", "update", "watch"]
-
-    ---
-    apiVersion: rbac.authorization.k8s.io/v1
-    kind: ClusterRoleBinding
-    metadata:
-      name: cluster-autoscaler
-      labels:
-        k8s-addon: cluster-autoscaler.addons.k8s.io
-        k8s-app: cluster-autoscaler
-    roleRef:
-      apiGroup: rbac.authorization.k8s.io
-      kind: ClusterRole
-      name: cluster-autoscaler
-    subjects:
-      - kind: ServiceAccount
-        name: cluster-autoscaler
-        namespace: kube-system
-
-    ---
-    apiVersion: rbac.authorization.k8s.io/v1
-    kind: RoleBinding
-    metadata:
-      name: cluster-autoscaler
-      namespace: kube-system
-      labels:
-        k8s-addon: cluster-autoscaler.addons.k8s.io
-        k8s-app: cluster-autoscaler
-    roleRef:
-      apiGroup: rbac.authorization.k8s.io
-      kind: Role
-      name: cluster-autoscaler
-    subjects:
-      - kind: ServiceAccount
-        name: cluster-autoscaler
-        namespace: kube-system
-
-    ---
-    apiVersion: apps/v1
-    kind: Deployment
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["events", "endpoints"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/eviction"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    resourceNames: ["cluster-autoscaler"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update"]
+  - apiGroups: [""]
+    resources:
+      - "pods"
+      - "services"
+      - "replicationcontrollers"
+      - "persistentvolumeclaims"
+      - "persistentvolumes"
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["extensions"]
+    resources: ["replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["watch", "list"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses", "csinodes"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["batch", "extensions"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create"]
+  - apiGroups: ["coordination.k8s.io"]
+    resourceNames: ["cluster-autoscaler"]
+    resources: ["leases"]
+    verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+    verbs: ["delete", "get", "update", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+  annotations:
+    cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
     metadata:
-      name: cluster-autoscaler
-      namespace: kube-system
       labels:
         app: cluster-autoscaler
       annotations:
-        cluster-autoscaler.kubernetes.io/safe-to-evict="false"
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '8085'
     spec:
-      replicas: 1
-      selector:
-        matchLabels:
-          app: cluster-autoscaler
-      template:
-        metadata:
-          labels:
-            app: cluster-autoscaler
-          annotations:
-            prometheus.io/scrape: 'true'
-            prometheus.io/port: '8085'
-        spec:
-          serviceAccountName: cluster-autoscaler
-          containers:
-            - image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.17.3
-              name: cluster-autoscaler
-              resources:
-                limits:
-                  cpu: 100m
-                  memory: 300Mi
-                requests:
-                  cpu: 100m
-                  memory: 300Mi
-              command:
-                - ./cluster-autoscaler
-                - --v=4
-                - --stderrthreshold=info
-                - --cloud-provider=aws
-                - --skip-nodes-with-local-storage=false
-                - --expander=least-waste
-                - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/fission-scale-test
-                - --balance-similar-node-groups
-                - --skip-nodes-with-system-pods=true
-              volumeMounts:
-                - name: ssl-certs
-                  mountPath: /etc/ssl/certs/ca-certificates.crt
-                  readOnly: true
-              imagePullPolicy: "Always"
-          volumes:
+      serviceAccountName: cluster-autoscaler
+      containers:
+        - image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.17.3
+          name: cluster-autoscaler
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - ./cluster-autoscaler
+            - --v=6
+            - --stderrthreshold=info
+            - --cloud-provider=aws
+            - --skip-nodes-with-local-storage=false
+            - --expander=least-waste
+            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/scale-test
+            - --balance-similar-node-groups
+            - --skip-nodes-with-system-pods=false
+          volumeMounts:
             - name: ssl-certs
-              hostPath:
-                path: "/etc/ssl/certs/ca-bundle.crt"
+              mountPath: /etc/ssl/certs/ca-certificates.crt
+              readOnly: true
+          imagePullPolicy: "Always"
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs/ca-bundle.crt"

scripts/cluster-setup/cluster.yaml

@@ -16,19 +16,20 @@ nodeGroups:
     maxSize: 3
     instancePrefix: fission
     volumeSize: 80
-    tags:
-      k8s.io/cluster-autoscaler/node-template/label/fission: fissioninfra
     ssh:
       allow: true
   - name: fission-functions
     labels:
-        role: functions
+      role: functions
     desiredCapacity: 2
+    instanceType: c4.4xlarge
     minSize: 2
     maxSize: 25
     instancePrefix: functions
     volumeSize: 100
     tags:
-        k8s.io/cluster-autoscaler/node-template/label/fission: functions  
+        k8s.io/cluster-autoscaler/node-template/label/role: functions  
+        k8s.io/cluster-autoscaler/fission-scale-test: owned
+        k8s.io/cluster-autoscaler/enabled: "true"
     ssh:
       allow: true

scripts/cluster-setup/setupcluster.sh

@@ -3,12 +3,13 @@
 set -euo pipefail
 
 echo "Starting Eksctl cluster setup"
-eksctl create cluster -f cluster.yaml
+# eksctl create cluster -f cluster.yaml # Till --asg-access is supported in YAML file, we need to use CLI
+eksctl create cluster --name scale-test --version 1.17 --managed --asg-access --region ap-south-1 --node-type=c4.4xlarge --nodes-min 3 --nodes-max 20 --ssh-access 
 
 echo "Getting nodes"
 kubectl get nodes
 
-echo "Installing Autoscaler"
+echo "Installing Autoscaler, Metric Server"
 kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.6/components.yaml
 kubectl apply -f cluster-as.yaml
 
@@ -19,6 +20,7 @@ helm install fission-metrics --namespace monitoring prometheus-community/kube-pr
   --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \
   --set prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues=false
 
+# helm install --namespace fission --name-template fission --set prometheus.enabled=false https://github.com/fission/fission/releases/download/1.10.0/fission-all-1.10.0.tgz
 # kubectl annotate svc/router -nfission service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600"
 # kubectl apply -f servicemonitors.yaml --namespace fission 
 # kubectl --namespace monitoring port-forward svc/fission-metrics-grafana 3000:80