From 88ab712b518bc7a3926ceb7f3d97a0d6263ac5e4 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 6 Oct 2020 17:53:37 +0200 Subject: [PATCH 01/13] Added HPA with custom metrics --- Deploy/demos/vnodes/BlackFriday.ps1 | 19 +++++++++++-------- Deploy/demos/vnodes/hpa/hpa-products.yaml | 16 ++++++++++++++++ .../vnodes/prometheus-adapter/values.yaml | 19 +++++++++++++++++++ Deploy/demos/vnodes/readme.md | 12 ++++++++++++ .../vnodes/service-monitor/sm-products.yaml | 12 ++++++++++++ .../demos/vnodes/yamls/1-service-account.yaml | 4 ++++ Deploy/demos/vnodes/yamls/2-cluster-role.yaml | 18 ++++++++++++++++++ .../vnodes/yamls/3-cluster-role-binding.yaml | 12 ++++++++++++ Deploy/demos/vnodes/yamls/4-prometheus.yaml | 13 +++++++++++++ Source/ApiGWs/Tailwind.Traders.Bff/Dockerfile | 4 ++-- .../ApiGWs/Tailwind.Traders.WebBff/Dockerfile | 4 ++-- .../Dockerfile | 4 ++-- .../Tailwind.Traders.Login.Api/Dockerfile | 4 ++-- .../Tailwind.Traders.Product.Api/Dockerfile | 4 ++-- .../Tailwind.Traders.Product.Api/Startup.cs | 3 +++ .../Tailwind.Traders.Product.Api.csproj | 2 ++ .../Tailwind.Traders.Profile.Api/Dockerfile | 4 ++-- Source/build/dependencies.props | 2 +- 18 files changed, 135 insertions(+), 21 deletions(-) create mode 100644 Deploy/demos/vnodes/hpa/hpa-products.yaml create mode 100644 Deploy/demos/vnodes/prometheus-adapter/values.yaml create mode 100644 Deploy/demos/vnodes/service-monitor/sm-products.yaml create mode 100644 Deploy/demos/vnodes/yamls/1-service-account.yaml create mode 100644 Deploy/demos/vnodes/yamls/2-cluster-role.yaml create mode 100644 Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml create mode 100644 Deploy/demos/vnodes/yamls/4-prometheus.yaml diff --git a/Deploy/demos/vnodes/BlackFriday.ps1 b/Deploy/demos/vnodes/BlackFriday.ps1 index 8e5184b5..14f1a7ce 100644 --- a/Deploy/demos/vnodes/BlackFriday.ps1 +++ b/Deploy/demos/vnodes/BlackFriday.ps1 @@ -1,5 +1,6 @@ Param( - [parameter(Mandatory=$true)][string]$aksName, + [parameter(Mandatory=$false)][string]$aksHost, + [parameter(Mandatory=$false)][string]$aksName, [parameter(Mandatory=$true)][string]$resourceGroup ) @@ -32,14 +33,16 @@ workflow LoadTestUri { } -$aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json) -if (-not $aksHost) { - $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json) -} +if ([String]::IsNullOrEmpty($aksHost)) { + $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpapplicationrouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json) + if (-not $aksHost) { + $aksHost=$(az aks show -n $aksName -g $resourceGroup -o json --query addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName | ConvertFrom-Json) + } -if (-not $aksHost) { - Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup" - exit 1 + if (-not $aksHost) { + Write-Host "Could not infer URL for AKS $aksName in RG $resourceGroup" + exit 1 + } } $url="http://$aksHost/product-api/v1/product" diff --git a/Deploy/demos/vnodes/hpa/hpa-products.yaml b/Deploy/demos/vnodes/hpa/hpa-products.yaml new file mode 100644 index 00000000..b71f538e --- /dev/null +++ b/Deploy/demos/vnodes/hpa/hpa-products.yaml @@ -0,0 +1,16 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: products-reqs-progress-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: tailwindtraders-product + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Pods + pods: + metricName: http_requests_in_progress + targetAverageValue: 3 \ No newline at end of file diff --git a/Deploy/demos/vnodes/prometheus-adapter/values.yaml b/Deploy/demos/vnodes/prometheus-adapter/values.yaml new file mode 100644 index 00000000..f8840f4c --- /dev/null +++ b/Deploy/demos/vnodes/prometheus-adapter/values.yaml @@ -0,0 +1,19 @@ +# Url to access prometheus +prometheus: + url: http://prometheus-operated.default.svc.cluster.local + port: 9090 + +metricsRelistInterval: 30s +logLevel: 6 +rules: + default: false + custom: + - seriesQuery: 'http_requests_in_progress{container="tailwindtraders-products"}' + resources: + overrides: + namespace: {resource: "namespace"} + pod: {resource: "pod"} + name: + as: "http_requests_in_progress" +# metricsQuery: 'round(sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[30s])))' + metricsQuery: 'http_requests_in_progress' diff --git a/Deploy/demos/vnodes/readme.md b/Deploy/demos/vnodes/readme.md index 0a75fbd2..2105e899 100644 --- a/Deploy/demos/vnodes/readme.md +++ b/Deploy/demos/vnodes/readme.md @@ -2,6 +2,18 @@ This folder contains additional scripts needed for testing the vnodes scenario + +## Pre steps + +1. Install prometheus operator: `kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/master/bundle.yaml` +2. Run prometheus instance: `kubectl apply -f yamls` +3. Deploy Prometheus Metrics Adapter: + * `helm repo add prometheus-community https://prometheus-community.github.io/helm-charts` + * `helm repo update` + * `helm install prometheus-adapter prometheus-community/prometheus-adapter -f ./prometheus-adapter/values.yaml` +4. Deploy the service monitor to monitor products api: `kubectl apply -f service-monitor` +5. Create the HPA for products: `kubectl apply -f hpa` + ## Powershell scripts - `Create-Aks.ps1`: Creates an AKS with vnodes enabled. diff --git a/Deploy/demos/vnodes/service-monitor/sm-products.yaml b/Deploy/demos/vnodes/service-monitor/sm-products.yaml new file mode 100644 index 00000000..fc37da15 --- /dev/null +++ b/Deploy/demos/vnodes/service-monitor/sm-products.yaml @@ -0,0 +1,12 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: sm-products + labels: + app: tailwindtraders +spec: + selector: + matchLabels: + app: tailwindtraders-products + endpoints: + - port: http \ No newline at end of file diff --git a/Deploy/demos/vnodes/yamls/1-service-account.yaml b/Deploy/demos/vnodes/yamls/1-service-account.yaml new file mode 100644 index 00000000..f671fc5a --- /dev/null +++ b/Deploy/demos/vnodes/yamls/1-service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus \ No newline at end of file diff --git a/Deploy/demos/vnodes/yamls/2-cluster-role.yaml b/Deploy/demos/vnodes/yamls/2-cluster-role.yaml new file mode 100644 index 00000000..299ebd0a --- /dev/null +++ b/Deploy/demos/vnodes/yamls/2-cluster-role.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] \ No newline at end of file diff --git a/Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml b/Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml new file mode 100644 index 00000000..db2717cd --- /dev/null +++ b/Deploy/demos/vnodes/yamls/3-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: default \ No newline at end of file diff --git a/Deploy/demos/vnodes/yamls/4-prometheus.yaml b/Deploy/demos/vnodes/yamls/4-prometheus.yaml new file mode 100644 index 00000000..d02d7626 --- /dev/null +++ b/Deploy/demos/vnodes/yamls/4-prometheus.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: prometheus + labels: + prometheus: prometheus +spec: + replicas: 1 + scrapeInterval: 5s + serviceAccountName: prometheus + serviceMonitorSelector: + matchLabels: + app: tailwindtraders \ No newline at end of file diff --git a/Source/ApiGWs/Tailwind.Traders.Bff/Dockerfile b/Source/ApiGWs/Tailwind.Traders.Bff/Dockerfile index 462abd86..79dbf749 100644 --- a/Source/ApiGWs/Tailwind.Traders.Bff/Dockerfile +++ b/Source/ApiGWs/Tailwind.Traders.Bff/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/ApiGWs/Tailwind.Traders.WebBff/Dockerfile b/Source/ApiGWs/Tailwind.Traders.WebBff/Dockerfile index 91d170b6..cd92c46c 100644 --- a/Source/ApiGWs/Tailwind.Traders.WebBff/Dockerfile +++ b/Source/ApiGWs/Tailwind.Traders.WebBff/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/Services/Tailwind.Traders.ImageClassifier.Api/Dockerfile b/Source/Services/Tailwind.Traders.ImageClassifier.Api/Dockerfile index bbd9c2d3..e9e98efc 100644 --- a/Source/Services/Tailwind.Traders.ImageClassifier.Api/Dockerfile +++ b/Source/Services/Tailwind.Traders.ImageClassifier.Api/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/Services/Tailwind.Traders.Login.Api/Dockerfile b/Source/Services/Tailwind.Traders.Login.Api/Dockerfile index 87ac42ee..bdccc9fd 100644 --- a/Source/Services/Tailwind.Traders.Login.Api/Dockerfile +++ b/Source/Services/Tailwind.Traders.Login.Api/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/Services/Tailwind.Traders.Product.Api/Dockerfile b/Source/Services/Tailwind.Traders.Product.Api/Dockerfile index 0534aecf..4e5033fd 100644 --- a/Source/Services/Tailwind.Traders.Product.Api/Dockerfile +++ b/Source/Services/Tailwind.Traders.Product.Api/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/Services/Tailwind.Traders.Product.Api/Startup.cs b/Source/Services/Tailwind.Traders.Product.Api/Startup.cs index c432f3fd..0e0f20ac 100644 --- a/Source/Services/Tailwind.Traders.Product.Api/Startup.cs +++ b/Source/Services/Tailwind.Traders.Product.Api/Startup.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Prometheus; using Tailwind.Traders.Product.Api.Extensions; namespace Tailwind.Traders.Product.Api @@ -66,11 +67,13 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) }); app.UseRouting(); + app.UseHttpMetrics(); app.UseEndpoints(endpoints => { endpoints.MapHealthChecks("/liveness", new HealthCheckOptions() { Predicate = r => r.Name.Contains("self") }); endpoints.MapHealthChecks("/readiness", new HealthCheckOptions() { }); + endpoints.MapMetrics(); endpoints.MapDefaultControllerRoute(); endpoints.MapControllers(); }); diff --git a/Source/Services/Tailwind.Traders.Product.Api/Tailwind.Traders.Product.Api.csproj b/Source/Services/Tailwind.Traders.Product.Api/Tailwind.Traders.Product.Api.csproj index 3b6934dc..3afb68e8 100644 --- a/Source/Services/Tailwind.Traders.Product.Api/Tailwind.Traders.Product.Api.csproj +++ b/Source/Services/Tailwind.Traders.Product.Api/Tailwind.Traders.Product.Api.csproj @@ -40,6 +40,8 @@ + + diff --git a/Source/Services/Tailwind.Traders.Profile.Api/Dockerfile b/Source/Services/Tailwind.Traders.Profile.Api/Dockerfile index de41ac40..f8441f54 100644 --- a/Source/Services/Tailwind.Traders.Profile.Api/Dockerfile +++ b/Source/Services/Tailwind.Traders.Profile.Api/Dockerfile @@ -1,5 +1,5 @@ -ARG sdkTag=3.0 -ARG runtimeTag=3.0 +ARG sdkTag=3.1 +ARG runtimeTag=3.1 ARG image=mcr.microsoft.com/dotnet/core/aspnet ARG sdkImage=mcr.microsoft.com/dotnet/core/sdk diff --git a/Source/build/dependencies.props b/Source/build/dependencies.props index 3109dc43..03f9c5ca 100644 --- a/Source/build/dependencies.props +++ b/Source/build/dependencies.props @@ -1,6 +1,6 @@ - netcoreapp3.0 + netcoreapp3.1 latest From 441c491fc4c1772874c011237ee546f63537a9f6 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 6 Oct 2020 18:09:34 +0200 Subject: [PATCH 02/13] Updated prometheus query on adapter --- Deploy/demos/vnodes/prometheus-adapter/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Deploy/demos/vnodes/prometheus-adapter/values.yaml b/Deploy/demos/vnodes/prometheus-adapter/values.yaml index f8840f4c..df82a415 100644 --- a/Deploy/demos/vnodes/prometheus-adapter/values.yaml +++ b/Deploy/demos/vnodes/prometheus-adapter/values.yaml @@ -15,5 +15,4 @@ rules: pod: {resource: "pod"} name: as: "http_requests_in_progress" -# metricsQuery: 'round(sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[30s])))' - metricsQuery: 'http_requests_in_progress' + metricsQuery: 'round(sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[30s])))' From 0b00aac732e92ab26ac283e83841bf26f4c8f813 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Fri, 9 Oct 2020 11:15:17 +0200 Subject: [PATCH 03/13] vnodes demo --- Deploy/demos/vnodes/blackfriday.sh | 32 +++++++++ .../vnodes/prometheus-adapter/values.yaml | 6 +- Deploy/demos/vnodes/readme.md | 67 ++++++++++++++++++- 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 Deploy/demos/vnodes/blackfriday.sh diff --git a/Deploy/demos/vnodes/blackfriday.sh b/Deploy/demos/vnodes/blackfriday.sh new file mode 100644 index 00000000..00c880fa --- /dev/null +++ b/Deploy/demos/vnodes/blackfriday.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +dns="" + +while [ "$1" != "" ]; do + case $1 in + -d | --dns) shift + dns=$1 + ;; + * ) echo "Invalid param. Use -d (ingress DNS)" + exit 1 + esac + shift +done + +if [ -z "$dns" ]; then + echo "No public ingress DNS provided. Use -d to provide it (no http prefix)" + exit 1 +fi + + +while : +do + id=$(shuf -i 1-100 -n 1) + if (($id > 50)); then + echo "# Calling http:$dns/product-api/v1/product" + curl -s "http://$dns/product-api/v1/product" > /dev/null + else + echo "! Calling http:$dns/product-api/v1/product/$id" + curl -s "http://$dns/product-api/v1/product/$id" > /dev/null + fi +done \ No newline at end of file diff --git a/Deploy/demos/vnodes/prometheus-adapter/values.yaml b/Deploy/demos/vnodes/prometheus-adapter/values.yaml index df82a415..b421293a 100644 --- a/Deploy/demos/vnodes/prometheus-adapter/values.yaml +++ b/Deploy/demos/vnodes/prometheus-adapter/values.yaml @@ -8,11 +8,11 @@ logLevel: 6 rules: default: false custom: - - seriesQuery: 'http_requests_in_progress{container="tailwindtraders-products"}' + - seriesQuery: 'http_requests_in_progress{namespace!="",pod!=""}' resources: overrides: namespace: {resource: "namespace"} pod: {resource: "pod"} name: - as: "http_requests_in_progress" - metricsQuery: 'round(sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[30s])))' + as: "http_requests_in_progress" + metricsQuery: 'sum(avg_over_time(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)' \ No newline at end of file diff --git a/Deploy/demos/vnodes/readme.md b/Deploy/demos/vnodes/readme.md index 2105e899..6b06396f 100644 --- a/Deploy/demos/vnodes/readme.md +++ b/Deploy/demos/vnodes/readme.md @@ -14,6 +14,69 @@ This folder contains additional scripts needed for testing the vnodes scenario 4. Deploy the service monitor to monitor products api: `kubectl apply -f service-monitor` 5. Create the HPA for products: `kubectl apply -f hpa` -## Powershell scripts +## Verify metrics server -- `Create-Aks.ps1`: Creates an AKS with vnodes enabled. +Verify metrics are exposed through metrics server: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/ | jq` + +```json +{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "custom.metrics.k8s.io/v1beta1", + "resources": [ + { + "name": "namespaces/http_requests_in_progress", + "singularName": "", + "namespaced": false, + "kind": "MetricValueList", + "verbs": [ + "get" + ] + }, + { + "name": "pods/http_requests_in_progress", + "singularName": "", + "namespaced": true, + "kind": "MetricValueList", + "verbs": [ + "get" + ] + } + ] +} +``` + +Verify metrics can be fetched from Prometheus: `kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress` + +```json +{ + "kind": "MetricValueList", + "apiVersion": "custom.metrics.k8s.io/v1beta1", + "metadata": { + "selfLink": "/apis/custom.metrics.k8s.io/v1beta1/namespace/default/http_requests_in_progress" + }, + "items": [ + { + "describedObject": { + "kind": "Namespace", + "name": "default", + "apiVersion": "/v1" + }, + "metricName": "http_requests_in_progress", + "timestamp": "2020-10-09T09:02:36Z", + "value": "1", + "selector": null + } + ] +} +``` + +## Run blackfriday scripts + +Now we need to do some calls to the products API. You can use the blackfriday script. Run several of them (`./blackfriday.sh -d /dev/null &`) and see how metric is growing in the HPA (`kubectl get hpa kubectl get hpa products-reqs-progress-hpa -w`). + +Once demo is finished kill the blackfriday scripts: + +`kill $(ps | grep blackfriday.sh | awk -F" " '{print $1}' )` + +After some time HPA will scale down the replicas. \ No newline at end of file From e2bbd49ea0db790a854f577033217f767345f82c Mon Sep 17 00:00:00 2001 From: eiximenis Date: Fri, 9 Oct 2020 12:16:01 +0200 Subject: [PATCH 04/13] Allowing deployment of products chart to vnodes, from both Deploy-Images-Aks.ps1 and Deploy-Unified.ps1 --- Deploy/Deploy-Unified.ps1 | 9 +++++++-- Deploy/demos/vnodes/null | 0 Deploy/helm/vnodes-values.yaml | 12 ++++++++++++ Deploy/powershell/Deploy-Images-Aks.ps1 | 5 +++++ 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 Deploy/demos/vnodes/null create mode 100644 Deploy/helm/vnodes-values.yaml diff --git a/Deploy/Deploy-Unified.ps1 b/Deploy/Deploy-Unified.ps1 index d8543532..2b0e99b0 100644 --- a/Deploy/Deploy-Unified.ps1 +++ b/Deploy/Deploy-Unified.ps1 @@ -6,7 +6,8 @@ Param( [parameter(Mandatory=$true)][string]$subscription, [parameter(Mandatory=$false)][string]$clientId, [parameter(Mandatory=$false)][string]$password, - [parameter(Mandatory=$false)][bool]$deployAks=$true + [parameter(Mandatory=$false)][bool]$deployAks=$true, + [parameter(Mandatory=$false)][bool]$deployProductsOnVnodes=$false ) $gValuesFile="configFile.yaml" @@ -49,7 +50,11 @@ Write-Host "The Name of your ACR: $acrName" -ForegroundColor Yellow # Deploy images in AKS $gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile) -& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts "*" -acrName $acrName -valuesFile $gValuesLocation +$chartsToDeploy = "*" +if ($deployProductsOnVnodes) { + $chartsToDeploy = "*,pr!" #pr! forces using vnodes for products chart +} +& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts $chartsToDeploy -acrName $acrName -valuesFile $gValuesLocation # Deploy pictures in AKS $storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name diff --git a/Deploy/demos/vnodes/null b/Deploy/demos/vnodes/null new file mode 100644 index 00000000..e69de29b diff --git a/Deploy/helm/vnodes-values.yaml b/Deploy/helm/vnodes-values.yaml new file mode 100644 index 00000000..2708f896 --- /dev/null +++ b/Deploy/helm/vnodes-values.yaml @@ -0,0 +1,12 @@ +## Add this values file to any Helm chart to force its deployment to run under virtual nodes + +tolerations: +- key: virtual-kubelet.io/provider + operator: Exists +- effect: NoSchedule + key: azure.com/aci + +nodeSelector: + kubernetes.io/role: agent + beta.kubernetes.io/os: linux + type: virtual-kubelet \ No newline at end of file diff --git a/Deploy/powershell/Deploy-Images-Aks.ps1 b/Deploy/powershell/Deploy-Images-Aks.ps1 index e5455866..a85a6e09 100644 --- a/Deploy/powershell/Deploy-Images-Aks.ps1 +++ b/Deploy/powershell/Deploy-Images-Aks.ps1 @@ -117,8 +117,13 @@ if ([String]::IsNullOrEmpty($valuesFile)) { Write-Host "Configuration file used is $valuesFile" -ForegroundColor Yellow if ($charts.Contains("pr") -or $charts.Contains("*")) { + $useVnodes = $charts.Contains("pr!") Write-Host "Products chart - pr" -ForegroundColor Yellow $command = "helm upgrade --install $name-product products-api -f $valuesFile --set ingress.hosts='{$aksHost}' --set image.repository=$acrLogin/product.api --set image.tag=$tag --set hpa.activated=$autoscale" + if ($useVnodes) { + Write-Host "Products chart - pr will run on virtual nodes" -ForegroundColor Yellow + $command = "$command -f vnodes-values.yaml" + } $command = createHelmCommand $command Invoke-Expression "$command" } From 8c95d8076e062aade2ff321659c3a20a46237b14 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 27 Oct 2020 14:30:17 +0100 Subject: [PATCH 05/13] installing appinsights az extension --- Deploy/Deploy-Unified.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Deploy/Deploy-Unified.ps1 b/Deploy/Deploy-Unified.ps1 index 2b0e99b0..5c753394 100644 --- a/Deploy/Deploy-Unified.ps1 +++ b/Deploy/Deploy-Unified.ps1 @@ -17,6 +17,9 @@ Push-Location $($MyInvocation.InvocationName | Split-Path) az extension add --name aks-preview az extension update --name aks-preview +az extension add --name application-insights +az extension update --name application-insights + # Write-Host "Login in your account" -ForegroundColor Yellow az login From 740cef80a9c0f8ac69fe75681273d8f54dd309e9 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 27 Oct 2020 16:57:17 +0100 Subject: [PATCH 06/13] update charts to not fail if not appinsights id is passed --- Deploy/helm/cart-api/templates/configmap.yaml | 2 +- Deploy/helm/coupons-api/templates/configmap.yaml | 2 +- Deploy/helm/image-classifier-api/templates/configmap.yaml | 2 +- Deploy/helm/login-api/templates/configmap.yaml | 2 +- Deploy/helm/mobilebff/templates/configmap.yaml | 2 +- Deploy/helm/popular-products-api/templates/configmap.yaml | 2 +- Deploy/helm/products-api/templates/configmap.yaml | 2 +- Deploy/helm/profiles-api/templates/configmap.yaml | 2 +- Deploy/helm/stock-api/templates/configmap.yaml | 2 +- Deploy/helm/webbff/templates/configmap.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Deploy/helm/cart-api/templates/configmap.yaml b/Deploy/helm/cart-api/templates/configmap.yaml index 8f6da2d0..1eb6bd69 100644 --- a/Deploy/helm/cart-api/templates/configmap.yaml +++ b/Deploy/helm/cart-api/templates/configmap.yaml @@ -21,6 +21,6 @@ data: policyName: {{ .Values.B2C.policyName | quote }} identityMetadata: {{ .Values.B2C.identityMetadata | quote }} issuer: {{ .Values.B2C.issuer | quote }} - APPLICATIONINSIGHTSIK: {{ .Values.inf.appinsights.id }} + APPLICATIONINSIGHTSIK: "{{ .Values.inf.appinsights.id }}" diff --git a/Deploy/helm/coupons-api/templates/configmap.yaml b/Deploy/helm/coupons-api/templates/configmap.yaml index f694634e..69c6029f 100644 --- a/Deploy/helm/coupons-api/templates/configmap.yaml +++ b/Deploy/helm/coupons-api/templates/configmap.yaml @@ -18,4 +18,4 @@ data: URL_BASE: {{ $imagebaseurl }} ISSUER: {{ .Values.sec.issuer }} SECURITYKEY: {{ .Values.sec.securityKey }} - APPLICATIONINSIGHTSIK: {{ .Values.inf.appinsights.id }} \ No newline at end of file + APPLICATIONINSIGHTSIK: "{{ .Values.inf.appinsights.id }}" \ No newline at end of file diff --git a/Deploy/helm/image-classifier-api/templates/configmap.yaml b/Deploy/helm/image-classifier-api/templates/configmap.yaml index 24499afa..0d9fa0ca 100644 --- a/Deploy/helm/image-classifier-api/templates/configmap.yaml +++ b/Deploy/helm/image-classifier-api/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" ImagePath: /imguploads ISSUER: {{ .Values.sec.issuer }} SECURITYKEY: {{ .Values.sec.securityKey }} diff --git a/Deploy/helm/login-api/templates/configmap.yaml b/Deploy/helm/login-api/templates/configmap.yaml index bad35aee..d4aa8cf4 100644 --- a/Deploy/helm/login-api/templates/configmap.yaml +++ b/Deploy/helm/login-api/templates/configmap.yaml @@ -19,4 +19,4 @@ data: ProfilesImageUrl: {{ .Values.inf.storage.profileimages }} Issuer: {{ .Values.sec.issuer }} SecurityKey: {{ .Values.sec.securityKey }} - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" diff --git a/Deploy/helm/mobilebff/templates/configmap.yaml b/Deploy/helm/mobilebff/templates/configmap.yaml index 8b4f14d0..071e3898 100644 --- a/Deploy/helm/mobilebff/templates/configmap.yaml +++ b/Deploy/helm/mobilebff/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" PopularProductsApiUrl: {{ .Values.inf.apiurls.popularproductsapiurl }} ProductsApiUrl: {{ .Values.inf.apiurls.productsapiurl }} ProfileApiUrl: {{ .Values.inf.apiurls.profileapiurl }} diff --git a/Deploy/helm/popular-products-api/templates/configmap.yaml b/Deploy/helm/popular-products-api/templates/configmap.yaml index 45f7021f..f1be758e 100644 --- a/Deploy/helm/popular-products-api/templates/configmap.yaml +++ b/Deploy/helm/popular-products-api/templates/configmap.yaml @@ -12,7 +12,7 @@ metadata: data: AUTHKEY: {{ .Values.inf.db.popularproducts.auth }} HOST: {{ .Values.inf.db.popularproducts.host }} - ApplicationInsightsIK: {{ .Values.inf.appinsights.id }} + ApplicationInsightsIK: "{{ .Values.inf.appinsights.id }}" AzureStorageUrl: {{ .Values.inf.storage.productimages }} ISSUER: {{ .Values.sec.issuer }} SECURITYKEY: {{ .Values.sec.securityKey }} diff --git a/Deploy/helm/products-api/templates/configmap.yaml b/Deploy/helm/products-api/templates/configmap.yaml index 02fb15b5..cefdd27a 100644 --- a/Deploy/helm/products-api/templates/configmap.yaml +++ b/Deploy/helm/products-api/templates/configmap.yaml @@ -14,6 +14,6 @@ data: CosmosDb__Host: {{ .Values.inf.db.products.host }} ProductImagesUrl: {{ .Values.inf.storage.productimages }} ProductDetailImagesUrl: {{ .Values.inf.storage.productdetailimages }} - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" ISSUER: {{ .Values.sec.issuer }} SECURITYKEY: {{ .Values.sec.securityKey }} diff --git a/Deploy/helm/profiles-api/templates/configmap.yaml b/Deploy/helm/profiles-api/templates/configmap.yaml index 36c5a962..2c8401b7 100644 --- a/Deploy/helm/profiles-api/templates/configmap.yaml +++ b/Deploy/helm/profiles-api/templates/configmap.yaml @@ -12,7 +12,7 @@ metadata: data: CosmosDb__Key: {{ .Values.inf.db.profile.auth }} CosmosDb__Host: {{ .Values.inf.db.profile.host }} - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" ProfilesImageUrl: {{ .Values.inf.storage.profileimages }} ISSUER: {{ .Values.sec.issuer }} SECURITYKEY: {{ .Values.sec.securityKey }} diff --git a/Deploy/helm/stock-api/templates/configmap.yaml b/Deploy/helm/stock-api/templates/configmap.yaml index 6ae356c8..c254cbd4 100644 --- a/Deploy/helm/stock-api/templates/configmap.yaml +++ b/Deploy/helm/stock-api/templates/configmap.yaml @@ -11,7 +11,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - APPLICATION_INSIGHTS_IKEY: {{ .Values.inf.appinsights.id }} + APPLICATION_INSIGHTS_IKEY: "{{ .Values.inf.appinsights.id }}" COSMOSDB_AUTHKEY: {{ .Values.inf.db.stock.auth }} COSMOSDB_HOST: {{ .Values.inf.db.stock.host }} ISSUER: {{ .Values.sec.issuer }} diff --git a/Deploy/helm/webbff/templates/configmap.yaml b/Deploy/helm/webbff/templates/configmap.yaml index ca783f94..2b0e9b3c 100644 --- a/Deploy/helm/webbff/templates/configmap.yaml +++ b/Deploy/helm/webbff/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} data: - ApplicationInsights__InstrumentationKey: {{ .Values.inf.appinsights.id }} + ApplicationInsights__InstrumentationKey: "{{ .Values.inf.appinsights.id }}" PopularProductsApiUrl: {{ .Values.inf.apiurls.popularproductsapiurl }} ProductsApiUrl: {{ .Values.inf.apiurls.productsapiurl }} ProfileApiUrl: {{ .Values.inf.apiurls.profileapiurl }} From 979514d6f3dac2c7cd97566b74a99d9022bc3fd9 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 27 Oct 2020 17:01:58 +0100 Subject: [PATCH 07/13] add parameters to avoid some steps --- Deploy/Deploy-Unified.ps1 | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/Deploy/Deploy-Unified.ps1 b/Deploy/Deploy-Unified.ps1 index 5c753394..803a80af 100644 --- a/Deploy/Deploy-Unified.ps1 +++ b/Deploy/Deploy-Unified.ps1 @@ -7,7 +7,11 @@ Param( [parameter(Mandatory=$false)][string]$clientId, [parameter(Mandatory=$false)][string]$password, [parameter(Mandatory=$false)][bool]$deployAks=$true, - [parameter(Mandatory=$false)][bool]$deployProductsOnVnodes=$false + [parameter(Mandatory=$false)][bool]$deployProductsOnVnodes=$false, + [parameter(Mandatory=$false)][bool]$stepDeployArm=$true, + [parameter(Mandatory=$false)][bool]$stepBuildPush=$true, + [parameter(Mandatory=$false)][bool]$stepDeployImages=$true, + [parameter(Mandatory=$false)][bool]$stepDeployPics=$true ) $gValuesFile="configFile.yaml" @@ -28,8 +32,10 @@ az account set --subscription $subscription Push-Location powershell -# Deploy ARM -& ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks +if ($stepDeployArm) { + # Deploy ARM + & ./Deploy-Arm-Azure.ps1 -resourceGroup $resourceGroup -location $location -clientId $clientId -password $password -deployAks $deployAks +} # Connecting kubectl to AKS Write-Host "Retrieving Aks Name" -ForegroundColor Yellow @@ -48,20 +54,26 @@ $acrName = $(az acr list --resource-group $resourceGroup --subscription $subscri Write-Host "The Name of your ACR: $acrName" -ForegroundColor Yellow & ./Create-Secret.ps1 -resourceGroup $resourceGroup -acrName $acrName -# Build an Push -& ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false +if ($stepBuildPush) { + # Build an Push + & ./Build-Push.ps1 -resourceGroup $resourceGroup -acrName $acrName -isWindows $false +} -# Deploy images in AKS -$gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile) -$chartsToDeploy = "*" -if ($deployProductsOnVnodes) { - $chartsToDeploy = "*,pr!" #pr! forces using vnodes for products chart +if ($stepDeployImages) { + # Deploy images in AKS + $gValuesLocation=$(./Join-Path-Recursively.ps1 -pathParts __values,$gValuesFile) + $chartsToDeploy = "*" + if ($deployProductsOnVnodes) { + $chartsToDeploy = "*,pr!" #pr! forces using vnodes for products chart + } + & ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts $chartsToDeploy -acrName $acrName -valuesFile $gValuesLocation } -& ./Deploy-Images-Aks.ps1 -aksName $aksName -resourceGroup $resourceGroup -charts $chartsToDeploy -acrName $acrName -valuesFile $gValuesLocation -# Deploy pictures in AKS -$storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name -& ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName +if ($stepDeployPics) { + # Deploy pictures in AKS + $storageName = $(az resource list --resource-group $resourceGroup --resource-type Microsoft.Storage/storageAccounts -o json | ConvertFrom-Json).name + & ./Deploy-Pictures-Azure.ps1 -resourceGroup $resourceGroup -storageName $storageName +} Pop-Location Pop-Location \ No newline at end of file From 6de2d52980f2446452fc9feee2c71a9e994b2ad0 Mon Sep 17 00:00:00 2001 From: eiximenis Date: Tue, 27 Oct 2020 17:07:59 +0100 Subject: [PATCH 08/13] param to avoid az login --- Deploy/Deploy-Unified.ps1 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Deploy/Deploy-Unified.ps1 b/Deploy/Deploy-Unified.ps1 index 803a80af..914bb325 100644 --- a/Deploy/Deploy-Unified.ps1 +++ b/Deploy/Deploy-Unified.ps1 @@ -11,7 +11,8 @@ Param( [parameter(Mandatory=$false)][bool]$stepDeployArm=$true, [parameter(Mandatory=$false)][bool]$stepBuildPush=$true, [parameter(Mandatory=$false)][bool]$stepDeployImages=$true, - [parameter(Mandatory=$false)][bool]$stepDeployPics=$true + [parameter(Mandatory=$false)][bool]$stepDeployPics=$true, + [parameter(Mandatory=$false)][bool]$stepLoginAzure=$true ) $gValuesFile="configFile.yaml" @@ -24,8 +25,10 @@ az extension update --name aks-preview az extension add --name application-insights az extension update --name application-insights -# Write-Host "Login in your account" -ForegroundColor Yellow -az login +if ($stepLoginAzure) { + # Write-Host "Login in your account" -ForegroundColor Yellow + az login +} # Write-Host "Choosing your subscription" -ForegroundColor Yellow az account set --subscription $subscription From 5ad377f9e253507d6c2f831293e4636b07253685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tom=C3=A0s?= Date: Thu, 29 Oct 2020 17:18:34 +0100 Subject: [PATCH 09/13] Updated limit to match request in product-api to allow vnodes deployment --- Deploy/helm/products-api/values.yaml | 2 +- Deploy/powershell/Generate-Config.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Deploy/helm/products-api/values.yaml b/Deploy/helm/products-api/values.yaml index 8a56a9c1..3b6ffcbc 100644 --- a/Deploy/helm/products-api/values.yaml +++ b/Deploy/helm/products-api/values.yaml @@ -14,7 +14,7 @@ resources: limits: cpu: "500m" requests: - cpu: "100m" + cpu: "500m" nodeSelector: "kubernetes.io/os": linux diff --git a/Deploy/powershell/Generate-Config.ps1 b/Deploy/powershell/Generate-Config.ps1 index 2c6af277..781ffb3c 100644 --- a/Deploy/powershell/Generate-Config.ps1 +++ b/Deploy/powershell/Generate-Config.ps1 @@ -75,7 +75,7 @@ if ($appInsightsName -and $appInsightsName.Length -eq 1) { $appinsightsId = $appinsightsConfig.instrumentationKey } } -Write-Host "App Insights Instrumentation Key: $($appinsightsId)" -ForegroundColor Yellow +Write-Host "App Insights Instrumentation Key: $appinsightsId" -ForegroundColor Yellow ## Showing Values that will be used From f3e75c136cca20ea89857cacad0c24fb5cafaadb Mon Sep 17 00:00:00 2001 From: cmcheca Date: Mon, 2 Nov 2020 18:53:36 +0100 Subject: [PATCH 10/13] Remove probe --- Deploy/helm/vnodes-values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Deploy/helm/vnodes-values.yaml b/Deploy/helm/vnodes-values.yaml index 2708f896..1a355b53 100644 --- a/Deploy/helm/vnodes-values.yaml +++ b/Deploy/helm/vnodes-values.yaml @@ -9,4 +9,6 @@ tolerations: nodeSelector: kubernetes.io/role: agent beta.kubernetes.io/os: linux - type: virtual-kubelet \ No newline at end of file + type: virtual-kubelet + +probes: {} \ No newline at end of file From 4a5769c6d7224b974c6f653d5ba5ee01dd6102cb Mon Sep 17 00:00:00 2001 From: cmcheca Date: Tue, 3 Nov 2020 12:08:21 +0100 Subject: [PATCH 11/13] Not execute probes when vnodes --- Deploy/helm/products-api/templates/deployment.yaml | 4 +++- Deploy/helm/vnodes-values.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Deploy/helm/products-api/templates/deployment.yaml b/Deploy/helm/products-api/templates/deployment.yaml index 002cf2a3..78947227 100644 --- a/Deploy/helm/products-api/templates/deployment.yaml +++ b/Deploy/helm/products-api/templates/deployment.yaml @@ -52,7 +52,8 @@ spec: - name: http containerPort: 80 protocol: TCP - {{ if .Values.probes -}} + {{ if not .Values.vnodes -}} + {{- if .Values.probes -}} {{- if .Values.probes.liveness -}} livenessProbe: httpGet: @@ -72,6 +73,7 @@ spec: periodSeconds: {{ .Values.probes.readiness.periodSeconds }} timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} {{- end -}} + {{- end -}} {{- end }} resources: {{ toYaml .Values.resources | indent 12 }} diff --git a/Deploy/helm/vnodes-values.yaml b/Deploy/helm/vnodes-values.yaml index 1a355b53..f495d4b4 100644 --- a/Deploy/helm/vnodes-values.yaml +++ b/Deploy/helm/vnodes-values.yaml @@ -11,4 +11,4 @@ nodeSelector: beta.kubernetes.io/os: linux type: virtual-kubelet -probes: {} \ No newline at end of file +vnodes: true \ No newline at end of file From defcee65e7f3c50ecb6c530e4abf808add6fa1b0 Mon Sep 17 00:00:00 2001 From: cmcheca Date: Tue, 3 Nov 2020 12:14:07 +0100 Subject: [PATCH 12/13] Test without parameter vnodes --- Deploy/helm/vnodes-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/helm/vnodes-values.yaml b/Deploy/helm/vnodes-values.yaml index f495d4b4..51c2f66c 100644 --- a/Deploy/helm/vnodes-values.yaml +++ b/Deploy/helm/vnodes-values.yaml @@ -11,4 +11,4 @@ nodeSelector: beta.kubernetes.io/os: linux type: virtual-kubelet -vnodes: true \ No newline at end of file +# vnodes: true \ No newline at end of file From 30bae715e368d3bee3a876d40e5f1617e5440afc Mon Sep 17 00:00:00 2001 From: cmcheca Date: Tue, 3 Nov 2020 12:34:14 +0100 Subject: [PATCH 13/13] Add parameter vnodes to remove probes --- Deploy/helm/vnodes-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy/helm/vnodes-values.yaml b/Deploy/helm/vnodes-values.yaml index 51c2f66c..f495d4b4 100644 --- a/Deploy/helm/vnodes-values.yaml +++ b/Deploy/helm/vnodes-values.yaml @@ -11,4 +11,4 @@ nodeSelector: beta.kubernetes.io/os: linux type: virtual-kubelet -# vnodes: true \ No newline at end of file +vnodes: true \ No newline at end of file