-
Notifications
You must be signed in to change notification settings - Fork 34
/
walk-the-dog.sh
executable file
·447 lines (390 loc) · 16.6 KB
/
walk-the-dog.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
mkdir -p outputs
export RG_NAME=$1
export LOCATION=$2
export SUFFIX=$3
export USERNAME=$4
export MONITORING=$5
export STATE_STORE=$6
export USE_VIRTUAL_CUSTOMER=$7
# set additional params
export UNIQUE_SERVICE_NAME=reddog$RANDOM$USERNAME$SUFFIX
start_time=$(date +%s)
# Check for Azure login
echo 'Checking to ensure logged into Azure CLI'
AZURE_LOGIN=0
# run a command against Azure to check if we are logged in already.
az group list -o table
# save the return code from above. Anything different than 0 means we need to login
AZURE_LOGIN=$?
if [[ ${AZURE_LOGIN} -ne 0 ]]; then
# not logged in. Initiate login process
az login --use-device-code
export AZURE_LOGIN
fi
# get current user
# export CURRENT_USER_ID=$(az ad signed-in-user show -o json | jq -r .objectId)
export CURRENT_USER_ID=$(az ad signed-in-user show -o json | jq -r .id)
# show all params
echo '****************************************************'
echo 'Starting Red Dog on AKS deployment'
echo ''
echo 'Parameters:'
echo 'SUBSCRIPTION: ' $SUBSCRIPTION_ID
echo 'TENANT: ' $TENANT_ID
echo 'LOCATION: ' $LOCATION
echo 'USER/PREFIX: ' $USERNAME
echo 'RG_NAME: ' $RG_NAME
echo 'UNIQUE NAME: ' $UNIQUE_SERVICE_NAME
echo 'LOGFILE_NAME: ' $LOGFILE_NAME
echo 'MONITORING: ' $MONITORING
echo 'STATE_STORE: ' $STATE_STORE
echo 'VIRTUAL CUSTOMER?: ' $USE_VIRTUAL_CUSTOMER
echo 'CURRENT_USER_ID: ' $CURRENT_USER_ID
echo '****************************************************'
echo ''
# update az CLI to install extensions automatically
az config set extension.use_dynamic_install=yes_without_prompt
# setup az CLI features
echo 'Configuring extensions and features for az CLI'
az feature register --namespace Microsoft.ContainerService --name AKS-ExtensionManager
az provider register --namespace Microsoft.Kubernetes --consent-to-permissions
az provider register --namespace Microsoft.ContainerService --consent-to-permissions
az provider register --namespace Microsoft.KubernetesConfiguration --consent-to-permissions
az extension add -n k8s-configuration
az extension add -n k8s-extension
az extension update -n k8s-configuration
az extension update -n k8s-extension
# create RG
echo "Creating Azure Resource Group"
az group create --name $RG_NAME --location $LOCATION
# create SSH keys
# echo 'Generating SSH keys (will overwrite existing)'
# ssh-keygen -f ~/.ssh/aks-reddog -N '' <<< y
# export SSH_PUB_KEY="$(cat ~/.ssh/aks-reddog.pub)"
# Bicep deployment
echo ''
echo '****************************************************'
echo 'Starting Bicep deployment of resources'
echo '****************************************************'
az deployment group create \
--name aks-reddog \
--mode Incremental \
--only-show-errors \
--resource-group $RG_NAME \
--template-file ./deploy/bicep/main.bicep \
--parameters uniqueServiceName=$UNIQUE_SERVICE_NAME \
--parameters currentUserId="$CURRENT_USER_ID" \
--parameters monitoringTool="$MONITORING" \
--parameters stateStore="$STATE_STORE"
# --parameters adminUsername="azureuser" \
# --parameters adminPublicKey="$SSH_PUB_KEY" \
echo ''
echo '****************************************************'
echo 'Base infra deployed. Starting config/app deployment'
echo '****************************************************'
# Save deployment outputs
az deployment group show -g $RG_NAME -n aks-reddog -o json --query properties.outputs > "./outputs/$RG_NAME-bicep-outputs.json"
# Connect to AKS and create namespace, redis
echo ''
echo '****************************************************'
echo 'AKS. Create namespace, secrets, Helm charts'
echo '****************************************************'
AKS_NAME=$(cat ./outputs/$RG_NAME-bicep-outputs.json | jq -r .aksName.value)
echo 'AKS Cluster Name: ' $AKS_NAME
az aks get-credentials -n $AKS_NAME -g $RG_NAME --overwrite-existing
echo ''
echo 'Create namespaces'
kubectl create ns reddog
kubectl create ns dapr-system
kubectl create ns traefik
echo ''
echo 'Helm repo updates'
helm repo add dapr https://dapr.github.io/helm-charts
helm repo add azure-marketplace https://marketplace.azurecr.io/helm/v1/repo
helm repo add traefik https://helm.traefik.io/traefik
helm repo update
echo ''
echo 'Deploying Dapr Helm chart'
helm install dapr dapr/dapr --namespace dapr-system
echo ''
echo 'Deploying Traefik Helm chart' #
helm install traefik traefik/traefik \
--namespace traefik \
--set service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"=reddog$SUFFIX
if [ "$STATE_STORE" = "redislocal" ]
then
echo ''
echo 'Deploying Redis Helm chart in AKS as configured in config.json' # https://bitnami.com/stack/redis/helm
export REDIS_PASSWD='w@lkingth3d0g'
kubectl create ns redis
helm install redis-release azure-marketplace/redis --namespace redis --set auth.password=$REDIS_PASSWD --set replica.replicaCount=2
kubectl create secret generic redis-password --from-literal=redis-password=$REDIS_PASSWD -n reddog
elif [ "$STATE_STORE" = "cosmos" ]
then
echo ''
echo 'CosmosDB deployed via Bicep as configured in config.json'
else
echo 'ERROR: State store value in config.json is not valid. Exiting'
exit 0
fi
# Zipkin
echo ''
echo 'Installing Zipkin for Dapr'
kubectl create ns zipkin
kubectl create deployment zipkin -n zipkin --image openzipkin/zipkin
kubectl expose deployment zipkin -n zipkin --type LoadBalancer --port 9411
if [ "$MONITORING" = "prometheus" ]
then
echo ''
echo 'Installing Prometheus/Grafana as configured in config.json'
echo 'Note: use admin to login'
git clone https://github.com/appdevgbb/kube-prometheus.git
cd kube-prometheus/manifests
kubectl apply --server-side -f ./setup
kubectl apply -f ./
cd ../..
rm -rf kube-prometheus
elif [ "$MONITORING" = "loganalytics" ]
then
echo ''
echo 'Log Analytics used for monitoring as configured in config.json'
else
echo 'No correct monitoring tool was set in config.json. Skipping'
fi
# Initialize KV
echo ''
echo 'Create SP for KV and setup permissions'
export KV_NAME=$(cat ./outputs/$RG_NAME-bicep-outputs.json | jq -r .keyvaultName.value)
echo 'Key Vault: ' $KV_NAME
az ad sp create-for-rbac \
--name "http://sp-$RG_NAME.microsoft.com" \
--only-show-errors \
--create-cert \
--cert $RG_NAME-cert \
--keyvault $KV_NAME \
--years 1
az ad sp create-for-rbac \
--name "http://sp-briantest.microsoft.com" \
--only-show-errors \
--create-cert \
--cert reddog-aks-4475-test-cert \
--keyvault reddog358briar4475 \
--years 1
## Get SP APP ID
echo 'Getting SP_APPID ...'
SP_INFO=$(az ad sp list -o json --display-name "http://sp-$RG_NAME.microsoft.com")
SP_APPID=$(echo $SP_INFO | jq -r .[].appId)
echo 'AKV SP_APPID: ' $SP_APPID
## Get SP Object ID
echo 'Getting SP_OBJECTID ...'
# SP_OBJECTID=$(echo $SP_INFO | jq -r .[].objectId)
SP_OBJECTID=$(echo $SP_INFO | jq -r .[].id)
echo 'AKV SP_OBJECTID: ' $SP_OBJECTID
# Assign SP to KV with GET permissions
az keyvault set-policy \
--name $KV_NAME \
--object-id $SP_OBJECTID \
--secret-permissions get \
--certificate-permissions get
# Assign permissions to the current user
UPN=$(az ad signed-in-user show -o json | jq -r '.userPrincipalName')
echo 'User UPN: ' $UPN
az keyvault set-policy \
--name $KV_NAME \
--secret-permissions get list set \
--certificate-permissions create get list \
--upn $UPN
# Download .pfx for Dapr secret (later)
az keyvault secret download \
--vault-name $KV_NAME \
--name $RG_NAME-cert \
--encoding base64 \
--file ./kv-$RG_NAME-cert.pfx
# Create K8s secret for above pfx (used by Dapr)
kubectl create secret generic reddog.secretstore \
--namespace reddog \
--from-file=secretstore-cert=./kv-$RG_NAME-cert.pfx \
--from-literal=vaultName=$KV_NAME \
--from-literal=spnClientId=$SP_APPID \
--from-literal=spnTenantId=$TENANT_ID
# Write keys to KV
echo ''
echo '****************************************************'
echo 'Writing all secrets to KeyVault'
echo '****************************************************'
# storage account
export STORAGE_NAME=$(cat ./outputs/$RG_NAME-bicep-outputs.json | jq -r .storageAccountName.value)
echo 'Storage Account: ' $STORAGE_NAME
export STORAGE_KEY=$(cat ./outputs/$RG_NAME-bicep-outputs.json | jq -r .storageAccountKey.value)
az keyvault secret set --vault-name $KV_NAME --name blob-storage-account --value $STORAGE_NAME
echo 'KeyVault secret created: blob-storage-account'
az keyvault secret set --vault-name $KV_NAME --name blob-storage-key --value $STORAGE_KEY
echo 'KeyVault secret created: blob-storage-key'
# service bus
export SB_NAME=$(jq -r .serviceBusName.value ./outputs/$RG_NAME-bicep-outputs.json)
export SB_CONNECT_STRING=$(jq -r .serviceBusConnectString.value ./outputs/$RG_NAME-bicep-outputs.json)
az keyvault secret set --vault-name $KV_NAME --name sb-root-connectionstring --value $SB_CONNECT_STRING
echo 'KeyVault secret created: sb-root-connectionstring'
# Azure SQL
export SQL_SERVER=$(jq -r .sqlServerName.value ./outputs/$RG_NAME-bicep-outputs.json)
export SQL_ADMIN_USER_NAME=$(jq -r .sqlAdmin.value ./outputs/$RG_NAME-bicep-outputs.json)
export SQL_ADMIN_PASSWD=$(jq -r .sqlPassword.value ./outputs/$RG_NAME-bicep-outputs.json)
export REDDOG_SQL_CONNECTION_STRING="Server=tcp:${SQL_SERVER}.database.windows.net,1433;Database=reddog;User ID=${SQL_ADMIN_USER_NAME};Password=${SQL_ADMIN_PASSWD};Encrypt=true;Connection Timeout=30;"
az keyvault secret set --vault-name $KV_NAME --name reddog-sql --value "${REDDOG_SQL_CONNECTION_STRING}"
echo 'KeyVault secret created: reddog-sql'
if [ "$STATE_STORE" = "redislocal" ]
then
echo ''
export REDIS_HOST='redis-release-master.redis.svc.cluster.local'
export REDIS_PORT='6379'
export REDIS_FQDN="${REDIS_HOST}:${REDIS_PORT}"
export REDIS_PASSWORD=$(kubectl get secret --namespace redis redis-release -o jsonpath="{.data.redis-password}" | base64 --decode)
az keyvault secret set --vault-name $KV_NAME --name redis-server --value $REDIS_FQDN
echo "KeyVault secret created: redis-server"
az keyvault secret set --vault-name $KV_NAME --name redis-password --value $REDIS_PASSWD
echo 'KeyVault secret created: redis-password'
elif [ "$STATE_STORE" = "cosmos" ]
then
echo ''
export COSMOS_URI=$(jq -r .cosmosUri.value ./outputs/$RG_NAME-bicep-outputs.json)
echo "Cosmos URI: " $COSMOS_URI
export COSMOS_ACCOUNT=$(jq -r .cosmosAccountName.value ./outputs/$RG_NAME-bicep-outputs.json)
echo "Cosmos Account: " $COSMOS_ACCOUNT
export COSMOS_PRIMARY_RW_KEY=$(az cosmosdb keys list -n $COSMOS_ACCOUNT -g $RG_NAME -o json | jq -r '.primaryMasterKey')
az keyvault secret set --vault-name $KV_NAME --name cosmos-uri --value $COSMOS_URI
echo "KeyVault secret created: cosmos-uri"
az keyvault secret set --vault-name $KV_NAME --name cosmos-primary-rw-key --value $COSMOS_PRIMARY_RW_KEY
echo "KeyVault secret created: cosmos-primary-rw-key"
else
echo 'ERROR: State store value in config.json is not valid'
fi
# Azure SQL server must set firewall to allow azure services
export AZURE_SQL_SERVER=$(jq -r .sqlServerName.value ./outputs/$RG_NAME-bicep-outputs.json)
echo ''
echo 'Allow Azure Services to access Azure SQL (Firewall)'
az sql server firewall-rule create \
--resource-group $RG_NAME \
--server $AZURE_SQL_SERVER \
--name AllowAzureServices \
--start-ip-address 0.0.0.0 \
--end-ip-address 0.0.0.0
# Configure AKS Flux v2 GitOps deployments
echo ''
echo '****************************************************'
echo 'Configure AKS Flux v2 GitOps deployments'
echo '****************************************************'
export AKS_NAME=$(jq -r .aksName.value ./outputs/$RG_NAME-bicep-outputs.json)
# Deploy Flux config for base dapr components
echo ''
echo 'Deploy Flux config for base dapr components'
az k8s-configuration flux create \
--resource-group $RG_NAME \
--cluster-name $AKS_NAME \
--cluster-type managedClusters \
--scope cluster \
--name reddog-components-base \
--namespace flux-system \
--url https://github.com/Azure/reddog-aks.git \
--branch main \
--kustomization name=kustomize path=./manifests/base/components-main prune=true
if [ "$STATE_STORE" = "redislocal" ]
then
# Deploy Flux config Dapr components for Redis state store
echo ''
echo 'Deploy Flux config Dapr components for Redis state store'
az k8s-configuration flux create \
--resource-group $RG_NAME \
--cluster-name $AKS_NAME \
--cluster-type managedClusters \
--scope cluster \
--name reddog-components-redis \
--namespace flux-system \
--url https://github.com/Azure/reddog-aks.git \
--branch main \
--kustomization name=kustomize path=./manifests/base/components-redis prune=true
elif [ "$STATE_STORE" = "cosmos" ]
then
# Deploy Flux config Dapr components for Cosmos state store
echo ''
echo 'Deploy Flux config Dapr components for Cosmos state store'
az k8s-configuration flux create \
--resource-group $RG_NAME \
--cluster-name $AKS_NAME \
--cluster-type managedClusters \
--scope cluster \
--name reddog-components-cosmos \
--namespace flux-system \
--url https://github.com/Azure/reddog-aks.git \
--branch main \
--kustomization name=kustomize path=./manifests/base/components-cosmos prune=true
else
echo 'ERROR: State store value in config.json is not valid'
fi
# Deploy Flux config for reddog base services
echo ''
echo 'Deploy Flux config for reddog base services'
az k8s-configuration flux create \
--resource-group $RG_NAME \
--cluster-name $AKS_NAME \
--cluster-type managedClusters \
--scope cluster \
--name reddog-deployments \
--namespace flux-system \
--url https://github.com/Azure/reddog-aks.git \
--branch main \
--kustomization name=kustomize path=./manifests/base/deployments prune=true
# Deploy Flux config for virtual customer (optional)
if [ "$USE_VIRTUAL_CUSTOMER" = "true" ]
then
echo ''
echo 'Deploy Flux config for virtual customer as specified in config.json'
az k8s-configuration flux create \
--resource-group $RG_NAME \
--cluster-name $AKS_NAME \
--cluster-type managedClusters \
--scope cluster \
--name reddog-virt-customer \
--namespace flux-system \
--url https://github.com/Azure/reddog-aks.git \
--branch main \
--kustomization name=kustomize path=./manifests/base/deployments-virt-customer prune=true
else
echo 'Virtual Customer was not deployed as specified in config.json'
fi
sleep 60
# get URL's for application
export UI_URL="http://"$(kubectl get svc --namespace reddog ui -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
export ORDER_URL="http://"$(kubectl get svc --namespace reddog order-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}')":8081"
export MAKE_LINE_URL="http://"$(kubectl get svc --namespace reddog make-line-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}')":8082"
export ACCOUNTING_URL="http://"$(kubectl get svc --namespace reddog accounting-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}')":8083"
export ZIPKIN_URL="http://"$(kubectl get svc --namespace zipkin zipkin -o jsonpath='{.status.loadBalancer.ingress[0].ip}')":9411"
if [ "$MONITORING" = "prometheus" ]
then
export GRAFANA_URL="http://"$(kubectl get svc --namespace monitoring grafana -o jsonpath='{.status.loadBalancer.ingress[0].ip}')":3000"
fi
echo ''
echo '*********************************************************************'
echo 'Application URLs'
echo ''
echo 'UI: ' $UI_URL
echo 'UI ingress path: ' 'http://reddog'$SUFFIX'.eastus.cloudapp.azure.com'
if [ "$MONITORING" = "prometheus" ]
then
echo 'Grafana dashboard (use admin): ' $GRAFANA_URL
fi
echo 'Zipkin: ' $ZIPKIN_URL
echo ''
echo 'Order test path: ' $ORDER_URL'/product'
echo 'Order test path (POST): ' $ORDER_URL'/order'
echo 'Makeline test path: ' $MAKE_LINE_URL'/orders/denver'
echo 'Accounting test path: ' $ACCOUNTING_URL'/OrderMetrics?StoreId=denver'
echo ''
echo 'Order Swagger: ' $ORDER_URL'/swagger/v1/swagger.json'
echo 'Makeline Swagger: ' $MAKE_LINE_URL'/swagger/v1/swagger.json'
echo 'Accounting Swagger: ' $ACCOUNTING_URL'/swagger/v1/swagger.json'
echo '*********************************************************************'
# elapsed time with second resolution
echo ''
end_time=$(date +%s)
elapsed=$(( end_time - start_time ))
printf 'Script elapsed time: %dh:%dm:%ds\n' $((elapsed/3600)) $((elapsed%3600/60)) $((elapsed%60))