Enable auto-scaling

deployment/bin/deploy

@@ -136,14 +136,22 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     setup_helm
 
+    # Install namespaces
+
+    echo "Installing namespaces..."
+
+    helm upgrade --install \
+        pc-namespaces helm/pc-namespaces \
+        -n pc-namespaces \
+        --create-namespace
+
     # Install cert-manager
 
     echo "Installing cert-manager..."
 
     helm upgrade --install \
         cert-manager \
         --namespace pc \
-        --create-namespace \
         --version v1.6.0 \
         --set installCRDs=true jetstack/cert-manager
 
@@ -193,6 +201,24 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --wait \
         --timeout 2m0s
 
+    echo "====================="
+    echo "==== Prometheus ====="
+    echo "====================="
+
+    echo "Deploying prometheus crd..."
+    kubectl apply -f helm/prometheus-crd --server-side
+
+    echo "Deploying prometheus component..."
+    kubectl apply -f helm/pc-apis-prometheus
+
+    echo "==========================="
+    echo "==== Overprovisioning ====="
+    echo "==========================="
+
+    helm upgrade --install overprovisioning helm/overprovisioning \
+        --kube-context "${KUBE_CONTEXT}" \
+        --wait \
+
     #########################
     # Deploy Azure Function #
     #########################

deployment/helm/deploy-values.template.yaml

@@ -102,10 +102,12 @@ pcingress:
       path: "/stac(/|$)(.*)"
       name: "planetary-computer-stac"
       port: "80"
+      blockMetrics: true
     tiler:
       path: "/data(/|$)(.*)"
       name: "planetary-computer-tiler"
       port: "80"
+      blockMetrics: true
 
   cert:
     secretName: "pqe-tls-secret"

deployment/helm/overprovisioning/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: planetary-computer-stac
+description: A Helm chart for the Overprovisioning virtual-nodes auxiliary
+type: application
+version: 0.1.1
+appVersion: 0.1.0

deployment/helm/overprovisioning/templates/overprovisioning.yaml

@@ -0,0 +1,118 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Values.namespace }}
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: overprovisioning
+value: -10
+globalDefault: false
+description: "Priority class used by overprovisioning."
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: cluster-proportional-autoscaler-overprovision
+  namespace: {{ .Values.namespace }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cluster-proportional-autoscaler-overprovision
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["replicationcontrollers/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: ["extensions","apps"]
+    resources: ["deployments/scale", "replicasets/scale"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cluster-proportional-autoscaler-overprovision
+subjects:
+  - kind: ServiceAccount
+    name: cluster-proportional-autoscaler-overprovision
+    namespace: {{ .Values.namespace }}
+roleRef:
+  kind: ClusterRole
+  name: cluster-proportional-autoscaler-overprovision
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: overprovisioning
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: overprovisioning
+  template:
+    metadata:
+      labels:
+        run: overprovisioning
+    spec:
+      priorityClassName: overprovisioning
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: reserve-resources
+        image: registry.k8s.io/pause:3.9
+        resources:
+          {{- toYaml .Values.overprovision.deployment.resources | nindent 10 }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: overprovisioning-autoscaler
+  namespace: {{ .Values.namespace }}
+data:
+  linear: |-
+    {
+      "coresPerReplica": {{ .Values.overprovision.hpa.coresPerReplica }},
+      "nodesPerReplica": {{ .Values.overprovision.hpa.nodesPerReplica }},
+      "min": {{ .Values.overprovision.hpa.minPods }},
+      "max": {{ .Values.overprovision.hpa.maxPods }},
+      "preventSinglePointFailure": {{ .Values.overprovision.hpa.preventSinglePointFailure }},
+      "includeUnschedulableNodes": {{ .Values.overprovision.hpa.includeUnschedulableNodes }}
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: overprovisioning-autoscaler
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: overprovisioning-autoscaler
+spec:
+  selector:
+    matchLabels:
+      app: overprovisioning-autoscaler
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: overprovisioning-autoscaler
+    spec:
+      containers:
+        - image: registry.k8s.io/cluster-proportional-autoscaler-amd64:1.8.1
+          name: autoscaler
+          command:
+            - /cluster-proportional-autoscaler
+            - --namespace={{ .Values.namespace }}
+            - --configmap=overprovisioning-autoscaler
+            - --default-params={"linear":{"coresPerReplica":8,"nodesPerReplica":4,"preventSinglePointFailure":false,"includeUnschedulableNodes":true}}
+            - --target=deployment/overprovisioning
+            - --logtostderr=true
+            - --v=2
+      serviceAccountName: cluster-proportional-autoscaler-overprovision

deployment/helm/overprovisioning/values.yaml

@@ -0,0 +1,18 @@
+environment: "staging"
+namespace: "overprovisioning"
+
+overprovision:
+
+  deployment:
+    resources:
+      requests:
+        memory: "512Mi"
+        cpu: "400m"
+
+  hpa:
+    coresPerReplica: 8
+    nodesPerReplica: 4
+    minPods: 1
+    maxPods: 3
+    preventSinglePointFailure: false
+    includeUnschedulableNodes: true

deployment/helm/pc-apis-ingress/templates/ingress.yaml

@@ -13,6 +13,12 @@ metadata:
   {{- with .Values.pcingress.ingress.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
+    {{- if .Values.pcingress.services.stac.blockMetrics -}}
+    nginx.ingress.kubernetes.io/server-snippet: |
+      if ($request_uri = {{ appRootPath }}/metrics) {
+          return 403;
+      }
+    {{- end }}
   {{- end }}
 spec:
   tls:

deployment/helm/pc-apis-prometheus/alertmanager-alertmanager.yaml

@@ -0,0 +1,36 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  labels:
+    app.kubernetes.io/component: alert-router
+    app.kubernetes.io/instance: main
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.25.0
+  name: main
+  namespace: monitoring
+spec:
+  image: quay.io/prometheus/alertmanager:v0.25.0
+  nodeSelector:
+    kubernetes.io/os: linux
+  podMetadata:
+    labels:
+      app.kubernetes.io/component: alert-router
+      app.kubernetes.io/instance: main
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/part-of: kube-prometheus
+      app.kubernetes.io/version: 0.25.0
+  replicas: 3
+  resources:
+    limits:
+      cpu: 100m
+      memory: 100Mi
+    requests:
+      cpu: 4m
+      memory: 100Mi
+  securityContext:
+    fsGroup: 2000
+    runAsNonRoot: true
+    runAsUser: 1000
+  serviceAccountName: alertmanager-main
+  version: 0.25.0

deployment/helm/pc-apis-prometheus/alertmanager-networkPolicy.yaml

@@ -0,0 +1,42 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  labels:
+    app.kubernetes.io/component: alert-router
+    app.kubernetes.io/instance: main
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.25.0
+  name: alertmanager-main
+  namespace: monitoring
+spec:
+  egress:
+  - {}
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          app.kubernetes.io/name: prometheus
+    ports:
+    - port: 9093
+      protocol: TCP
+    - port: 8080
+      protocol: TCP
+  - from:
+    - podSelector:
+        matchLabels:
+          app.kubernetes.io/name: alertmanager
+    ports:
+    - port: 9094
+      protocol: TCP
+    - port: 9094
+      protocol: UDP
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/component: alert-router
+      app.kubernetes.io/instance: main
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/part-of: kube-prometheus
+  policyTypes:
+  - Egress
+  - Ingress

deployment/helm/pc-apis-prometheus/alertmanager-podDisruptionBudget.yaml

@@ -0,0 +1,19 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    app.kubernetes.io/component: alert-router
+    app.kubernetes.io/instance: main
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 0.25.0
+  name: alertmanager-main
+  namespace: monitoring
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: alert-router
+      app.kubernetes.io/instance: main
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/part-of: kube-prometheus