Merge pull request #98 from microsoft/custom-metrics-hpa

Custom metrics hpa

Deploy/Deploy-Unified.ps1

@@ -6,7 +6,13 @@ Param(
     [parameter(Mandatory=$true)][string]$subscription,
     [parameter(Mandatory=$false)][string]$clientId,
     [parameter(Mandatory=$false)][string]$password,
-    [parameter(Mandatory=$false)][bool]$deployAks=$true
+    [parameter(Mandatory=$false)][bool]$deployAks=$true,
+    [parameter(Mandatory=$false)][bool]$deployProductsOnVnodes=$false,
+    [parameter(Mandatory=$false)][bool]$stepDeployArm=$true,
+    [parameter(Mandatory=$false)][bool]$stepBuildPush=$true,
+    [parameter(Mandatory=$false)][bool]$stepDeployImages=$true,
+    [parameter(Mandatory=$false)][bool]$stepDeployPics=$true,
+    [parameter(Mandatory=$false)][bool]$stepLoginAzure=$true
 )
 $gValuesFile="configFile.yaml"
 
@@ -16,16 +22,23 @@ Push-Location $($MyInvocation.InvocationName | Split-Path)
 az extension add --name aks-preview
 az extension update --name aks-preview
 
-# Write-Host "Login in your account" -ForegroundColor Yellow
-az login
+az extension add --name  application-insights
+az extension update --name  application-insights
+
+if ($stepLoginAzure) {
+    # Write-Host "Login in your account" -ForegroundColor Yellow
+    az login
+}
 
 # Write-Host "Choosing your subscription" -ForegroundColor Yellow
 az account set --subscription $subscription
 
 Push-Location powershell
 
-# Deploy ARM
-& ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks
+if ($stepDeployArm) {
+    # Deploy ARM
+    & ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks
+}
 
 # Connecting kubectl to AKS
 Write-Host "Retrieving Aks Name" -ForegroundColor Yellow
@@ -44,16 +57,26 @@ $acrName = $(az acr list --resource-group $resourceGroup --subscription $subscri
 Write-Host "The Name of your ACR: $acrName" -ForegroundColor Yellow
 & ./Create-Secret.ps1 -resourceGroup $resourceGroup -acrName $acrName
 
-# Build an Push
-& ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false
+if ($stepBuildPush) {
+    # Build an Push
+    & ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false
+}
 
-# Deploy images in AKS
-$gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile)
-& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts "*" -acrName $acrName -valuesFile $gValuesLocation
+if ($stepDeployImages) {
+    # Deploy images in AKS
+    $gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile)
+    $chartsToDeploy = "*"
+    if ($deployProductsOnVnodes) {
+        $chartsToDeploy = "*,pr!"           #pr! forces using vnodes for products chart
+    }
+    & ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts $chartsToDeploy -acrName $acrName -valuesFile $gValuesLocation
+}
 
-# Deploy pictures in AKS
-$storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name
-& ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName
+if ($stepDeployPics) {
+    # Deploy pictures in AKS
+    $storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name
+    & ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName
+}
 
 Pop-Location
 Pop-Location

Deploy/demos/vnodes/BlackFriday.ps1

@@ -1,5 +1,6 @@
 Param(
-    [parameter(Mandatory=$true)][string]$aksName,
+    [parameter(Mandatory=$false)][string]$aksHost,
+    [parameter(Mandatory=$false)][string]$aksName,
     [parameter(Mandatory=$true)][string]$resourceGroup
 )
 
@@ -32,14 +33,16 @@ workflow LoadTestUri {
 }
 
 
-$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
-if (-not $aksHost) {
-    $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
-}
+if ([String]::IsNullOrEmpty($aksHost)) {
+    $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
+    if (-not $aksHost) {
+        $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json)
+    }
 
-if (-not $aksHost) {
-    Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup"
-    exit 1
+    if (-not $aksHost) {
+        Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup"
+        exit 1
+    }
 }
 
 $url="http://$aksHost/product-api/v1/product"

Deploy/demos/vnodes/blackfriday.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+dns=""
+
+while [ "$1" != "" ]; do
+    case $1 in
+        -d | --dns)                     shift
+                                        dns=$1
+                                        ;;
+       * )                              echo "Invalid param. Use -d (ingress DNS)"
+                                        exit 1
+    esac
+    shift
+done
+
+if [ -z "$dns" ]; then
+    echo "No public ingress DNS provided. Use -d to provide it (no http prefix)"
+    exit 1
+fi
+
+
+while :
+do 
+    id=$(shuf -i 1-100 -n 1)
+    if (($id > 50)); then
+        echo "# Calling http:$dns/product-api/v1/product"
+        curl -s "http://$dns/product-api/v1/product" > /dev/null 
+    else
+        echo "! Calling http:$dns/product-api/v1/product/$id"
+        curl -s "http://$dns/product-api/v1/product/$id" > /dev/null
+    fi
+done

Deploy/demos/vnodes/hpa/hpa-products.yaml

@@ -0,0 +1,16 @@
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: products-reqs-progress-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: tailwindtraders-product
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+  - type: Pods
+    pods:
+      metricName: http_requests_in_progress
+      targetAverageValue: 3

Deploy/demos/vnodes/prometheus-adapter/values.yaml

@@ -0,0 +1,18 @@
+# Url to access prometheus
+prometheus:
+  url: http://prometheus-operated.default.svc.cluster.local
+  port: 9090
+
+metricsRelistInterval: 30s
+logLevel: 6
+rules:
+  default: false
+  custom:
+  - seriesQuery: 'http_requests_in_progress{namespace!="",pod!=""}'
+    resources:
+      overrides:
+        namespace: {resource: "namespace"}
+        pod: {resource: "pod"}
+    name:
+      as: "http_requests_in_progress"
+    metricsQuery: 'sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'

Deploy/demos/vnodes/readme.md

@@ -2,6 +2,81 @@
 
 This folder contains additional scripts needed for testing the vnodes scenario
 
-## Powershell scripts
 
-- `Create-Aks.ps1`: Creates an AKS with vnodes enabled.
+## Pre steps
+
+1. Install prometheus operator: `kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/bundle.yaml`
+2. Run prometheus instance: `kubectl apply -f yamls`
+3. Deploy Prometheus Metrics Adapter:
+    * `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts`
+    * `helm repo update`
+    * `helm install prometheus-adapter prometheus-community/prometheus-adapter  -f ./prometheus-adapter/values.yaml`
+4. Deploy the service monitor to monitor products api: `kubectl apply -f service-monitor`
+5. Create the HPA for products: `kubectl apply -f hpa`
+
+## Verify metrics server
+
+Verify metrics are exposed through metrics server: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/ | jq` 
+
+```json
+{
+  "kind": "APIResourceList",
+  "apiVersion": "v1",
+  "groupVersion": "custom.metrics.k8s.io/v1beta1",
+  "resources": [
+    {
+      "name": "namespaces/http_requests_in_progress",
+      "singularName": "",
+      "namespaced": false,
+      "kind": "MetricValueList",
+      "verbs": [
+        "get"
+      ]
+    },
+    {
+      "name": "pods/http_requests_in_progress",
+      "singularName": "",
+      "namespaced": true,
+      "kind": "MetricValueList",
+      "verbs": [
+        "get"
+      ]
+    }
+  ]
+}
+```
+
+Verify metrics can be fetched from Prometheus: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress`
+
+```json
+{
+  "kind": "MetricValueList",
+  "apiVersion": "custom.metrics.k8s.io/v1beta1",
+  "metadata": {
+    "selfLink": "/apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress"
+  },
+  "items": [
+    {
+      "describedObject": {
+        "kind": "Namespace",
+        "name": "default",
+        "apiVersion": "/v1"
+      },
+      "metricName": "http_requests_in_progress",
+      "timestamp": "2020-10-09T09:02:36Z",
+      "value": "1",
+      "selector": null
+    }
+  ]
+}
+```
+
+## Run blackfriday scripts
+
+Now we need to do some calls to the products API. You can use the  blackfriday script. Run several of them (`./blackfriday.sh -d <dns-ingress> /dev/null &`) and see how metric is growing in the HPA (`kubectl get hpa kubectl get hpa products-reqs-progress-hpa -w`).
+
+Once demo is finished kill the blackfriday scripts:
+
+`kill $(ps | grep blackfriday.sh |  awk -F" " '{print $1}' )`
+
+After some time HPA will scale down the replicas.

Deploy/demos/vnodes/service-monitor/sm-products.yaml

@@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name:  sm-products
+  labels:
+    app: tailwindtraders
+spec:
+  selector:
+    matchLabels:
+      app: tailwindtraders-products
+  endpoints:
+  - port: http

Deploy/demos/vnodes/yamls/1-service-account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus

Deploy/demos/vnodes/yamls/2-cluster-role.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources:
+  - configmaps
+  verbs: ["get"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]

Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: default

Deploy/demos/vnodes/yamls/4-prometheus.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: prometheus
+  labels:
+    prometheus: prometheus
+spec:
+  replicas: 1
+  scrapeInterval: 5s
+  serviceAccountName: prometheus
+  serviceMonitorSelector:
+    matchLabels:
+      app: tailwindtraders

Deploy/helm/cart-api/templates/configmap.yaml

@@ -21,6 +21,6 @@ data:
   policyName: {{ .Values.B2C.policyName | quote }}
   identityMetadata: {{ .Values.B2C.identityMetadata | quote }}
   issuer: {{ .Values.B2C.issuer | quote }}
-  APPLICATIONINSIGHTSIK:  {{ .Values.inf.appinsights.id }} 
+  APPLICATIONINSIGHTSIK:  "{{ .Values.inf.appinsights.id }}"
 
 

Deploy/helm/coupons-api/templates/configmap.yaml

@@ -18,4 +18,4 @@ data:
   URL_BASE: {{ $imagebaseurl }}
   ISSUER: {{ .Values.sec.issuer }}
   SECURITYKEY: {{ .Values.sec.securityKey }}
-  APPLICATIONINSIGHTSIK:  {{ .Values.inf.appinsights.id }} 
+  APPLICATIONINSIGHTSIK:  "{{ .Values.inf.appinsights.id }}" 

Deploy/helm/image-classifier-api/templates/configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 data:
-  ApplicationInsights__InstrumentationKey:  {{ .Values.inf.appinsights.id }}
+  ApplicationInsights__InstrumentationKey:  "{{ .Values.inf.appinsights.id }}"
   ImagePath: /imguploads
   ISSUER: {{ .Values.sec.issuer }}
   SECURITYKEY: {{ .Values.sec.securityKey }}

Deploy/helm/login-api/templates/configmap.yaml

@@ -19,4 +19,4 @@ data:
   ProfilesImageUrl: {{  .Values.inf.storage.profileimages  }}
   Issuer: {{ .Values.sec.issuer }}
   SecurityKey: {{ .Values.sec.securityKey }}
-  ApplicationInsights__InstrumentationKey:  {{ .Values.inf.appinsights.id }} 
+  ApplicationInsights__InstrumentationKey:  "{{ .Values.inf.appinsights.id }}" 

Deploy/helm/mobilebff/templates/configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 data:
-  ApplicationInsights__InstrumentationKey:  {{ .Values.inf.appinsights.id }}
+  ApplicationInsights__InstrumentationKey:  "{{ .Values.inf.appinsights.id }}"
   PopularProductsApiUrl: {{ .Values.inf.apiurls.popularproductsapiurl }}
   ProductsApiUrl: {{ .Values.inf.apiurls.productsapiurl }}
   ProfileApiUrl: {{ .Values.inf.apiurls.profileapiurl }}

Deploy/helm/popular-products-api/templates/configmap.yaml

@@ -12,7 +12,7 @@ metadata:
 data:
   AUTHKEY: {{ .Values.inf.db.popularproducts.auth }}
   HOST: {{ .Values.inf.db.popularproducts.host }}
-  ApplicationInsightsIK:  {{ .Values.inf.appinsights.id }}
+  ApplicationInsightsIK:  "{{ .Values.inf.appinsights.id }}"
   AzureStorageUrl: {{ .Values.inf.storage.productimages }}
   ISSUER: {{ .Values.sec.issuer }}
   SECURITYKEY: {{ .Values.sec.securityKey }}

Deploy/helm/products-api/templates/configmap.yaml

@@ -14,6 +14,6 @@ data:
   CosmosDb__Host: {{ .Values.inf.db.products.host }}
   ProductImagesUrl: {{ .Values.inf.storage.productimages }}
   ProductDetailImagesUrl: {{ .Values.inf.storage.productdetailimages }}
-  ApplicationInsights__InstrumentationKey:  {{ .Values.inf.appinsights.id }}  
+  ApplicationInsights__InstrumentationKey:  "{{ .Values.inf.appinsights.id }}"  
   ISSUER: {{ .Values.sec.issuer }}
   SECURITYKEY: {{ .Values.sec.securityKey }}