Repository navigation
Expand file tree
/
Copy pathquery-telemetry.sh
More file actions
executable file
·275 lines (242 loc) · 11.1 KB
/
Copy pathquery-telemetry.sh
File metadata and controls
executable file
·275 lines (242 loc) · 11.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
#!/bin/bash
# Usage: query-telemetry.sh
#
# Waits for the web app to answer (the deploy scripts upload it asynchronously), calls it, adds an activity, then reads
# back what the Microsoft OpenTelemetry Distro sent through the connection string of the Application Insights resource
# to its Log Analytics workspace: the requests the web app served (AppRequests), its calls to App Configuration, Key
# Vault and PostgreSQL (AppDependencies), the application's own log records (AppTraces), the custom event of the
# activity (AppEvents), the custom metric that counts the activities (AppMetrics), and the exceptions the application
# logged, if any (AppExceptions).
#
# The records reach the workspace within seconds on the emulator. On Azure they take minutes, so the wait and every
# query are retried: raise RETRY_COUNT there.
# Variables (the same defaults and environment overrides as deploy.sh)
PREFIX="${PREFIX:-local}"
SUFFIX="${SUFFIX:-test}"
RESOURCE_GROUP_NAME="${PREFIX}-rg"
WEB_APP_NAME="${PREFIX}-webapp-${SUFFIX}"
LOG_ANALYTICS_NAME="${PREFIX}-log-analytics-${SUFFIX}"
WEB_APP_CALLS="${WEB_APP_CALLS:-5}"
RETRY_COUNT="${RETRY_COUNT:-30}"
SLEEP="${SLEEP:-10}"
EXIT_CODE=0
# Run a KQL query on the Log Analytics workspace and print the records it returns as a table, retrying while the
# records are still on their way to the workspace
# $1: what the query reads
# $2: the KQL query
# $3: the JMESPath projection that names and orders the columns of the table
# $4: the number of records the query must return (1 when not given)
query_workspace() {
local description=$1
local query=$2
local columns=$3
local minimum_records=${4:-1}
local records
local attempt=1
echo -e "\n$description:\n"
while [ $attempt -le $RETRY_COUNT ]; do
records=$(az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "length(@)" \
--output tsv \
--only-show-errors)
if [[ $? == 0 && $records -ge $minimum_records ]]; then
az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "$columns" \
--output table \
--only-show-errors
if [[ $? != 0 ]]; then
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace did not answer the query that shows the records"
EXIT_CODE=1
fi
return 0
else
if [ $attempt -lt $RETRY_COUNT ]; then
echo "Attempt $attempt of $RETRY_COUNT: [${records:-0}] of the [$minimum_records] records expected so far. Waiting [$SLEEP] seconds before retry..."
sleep $SLEEP
fi
attempt=$((attempt + 1))
fi
done
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace returned [${records:-0}] of the [$minimum_records] records expected after $RETRY_COUNT attempts."
echo "On Azure the first records of a new component can take several minutes: run the script again later or raise RETRY_COUNT."
EXIT_CODE=1
}
# Run a KQL query on the Log Analytics workspace once and print the records it returns as a table, or say that there
# are none: for records that only an unexpected event produces
# $1: what the query reads
# $2: the KQL query
# $3: the JMESPath projection that names and orders the columns of the table
# $4: what it means when there are no records
show_workspace_records() {
local description=$1
local query=$2
local columns=$3
local when_empty=$4
local records
echo -e "\n$description:\n"
records=$(az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "length(@)" \
--output tsv \
--only-show-errors)
if [[ $? != 0 ]]; then
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace did not answer the query"
EXIT_CODE=1
elif [[ $records == 0 ]]; then
echo "$when_empty"
else
az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "$columns" \
--output table \
--only-show-errors
fi
}
# Check whether the log-analytics extension of the Azure CLI, which provides the az monitor log-analytics query
# command, is installed
echo "Checking if the [log-analytics] extension of the Azure CLI is installed..."
az extension show --name log-analytics --only-show-errors &>/dev/null
if [[ $? != 0 ]]; then
echo "No [log-analytics] extension of the Azure CLI is actually installed"
echo "Installing the [log-analytics] extension of the Azure CLI..."
# Install the log-analytics extension
az extension add \
--name log-analytics \
--only-show-errors 1>/dev/null
if [[ $? == 0 ]]; then
echo "[log-analytics] extension of the Azure CLI successfully installed"
else
echo "Failed to install the [log-analytics] extension of the Azure CLI"
exit 1
fi
else
echo "[log-analytics] extension of the Azure CLI already installed"
fi
# Get the workspace id (customer id) of the Log Analytics workspace, which identifies it to the query API
echo "Retrieving the workspace id of the [$LOG_ANALYTICS_NAME] Log Analytics workspace..."
WORKSPACE_ID=$(az monitor log-analytics workspace show \
--resource-group "$RESOURCE_GROUP_NAME" \
--workspace-name "$LOG_ANALYTICS_NAME" \
--query customerId \
--output tsv \
--only-show-errors)
if [[ -n $WORKSPACE_ID ]]; then
echo "Workspace id [$WORKSPACE_ID] of the [$LOG_ANALYTICS_NAME] Log Analytics workspace successfully retrieved"
else
echo "Failed to retrieve the workspace id of the [$LOG_ANALYTICS_NAME] Log Analytics workspace"
exit 1
fi
# Get the default host name of the web app
echo "Retrieving the default host name of the [$WEB_APP_NAME] web app..."
WEB_APP_HOST_NAME=$(az webapp show \
--name "$WEB_APP_NAME" \
--resource-group "$RESOURCE_GROUP_NAME" \
--query defaultHostName \
--output tsv \
--only-show-errors)
if [[ -n $WEB_APP_HOST_NAME ]]; then
echo "Default host name [$WEB_APP_HOST_NAME] of the [$WEB_APP_NAME] web app successfully retrieved"
else
echo "Failed to retrieve the default host name of the [$WEB_APP_NAME] web app"
exit 1
fi
# Wait until the web app answers: the deployment builds it asynchronously, and only a running app sends telemetry
echo "Waiting for the [$WEB_APP_NAME] web app to answer..."
ATTEMPT=1
while [ $ATTEMPT -le $RETRY_COUNT ]; do
STATUS_CODE=$(curl --max-time 30 --silent --output /dev/null --write-out "%{http_code}" "https://$WEB_APP_HOST_NAME/")
if [[ $STATUS_CODE == 200 ]]; then
echo "The [$WEB_APP_NAME] web app answered with status code [$STATUS_CODE]"
break
fi
if [ $ATTEMPT -lt $RETRY_COUNT ]; then
echo "Attempt $ATTEMPT of $RETRY_COUNT: the [$WEB_APP_NAME] web app answered with status code [$STATUS_CODE]. Waiting [$SLEEP] seconds before retry..."
sleep $SLEEP
fi
ATTEMPT=$((ATTEMPT + 1))
done
if [[ $STATUS_CODE != 200 ]]; then
echo "ERROR: the [$WEB_APP_NAME] web app answered with status code [$STATUS_CODE] after $RETRY_COUNT attempts"
exit 1
fi
# Call the web app, so that AppRequests, AppDependencies and AppTraces hold the activity of this run
echo "Calling the [$WEB_APP_NAME] web app [$WEB_APP_CALLS] times..."
for CALL in $(seq 1 "$WEB_APP_CALLS"); do
STATUS_CODE=$(curl --max-time 30 --silent --output /dev/null --write-out "%{http_code}" "https://$WEB_APP_HOST_NAME/")
echo "Call $CALL of $WEB_APP_CALLS to the [$WEB_APP_NAME] web app answered with status code [$STATUS_CODE]"
done
# Add an activity, so that AppEvents holds its ActivityAdded custom event and AppMetrics the activities custom metric.
# The form of the .NET app carries an antiforgery token, without which ASP.NET Core refuses the post: the script reads
# the form first and sends the token back with the cookie that came with it. The Python app has no such token. Right
# after a deployment, Azure answered the post with 405 before the app served it (2026-11-17), so the post is retried
# until the app stores the activity.
ACTIVITY="Telemetry check $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Adding the [$ACTIVITY] activity with the [$WEB_APP_NAME] web app..."
ATTEMPT=1
while [ $ATTEMPT -le $RETRY_COUNT ]; do
COOKIE_JAR=$(mktemp)
FORM_TOKEN=$(curl --max-time 30 --silent --cookie-jar "$COOKIE_JAR" "https://$WEB_APP_HOST_NAME/" |
grep -o 'name="__RequestVerificationToken"[^>]*value="[^"]*"' |
head -n 1 |
sed 's/.*value="//; s/"$//')
STATUS_CODE=$(curl --max-time 30 --silent --output /dev/null --write-out "%{http_code}" \
--cookie "$COOKIE_JAR" \
--data-urlencode "activity=$ACTIVITY" \
--data-urlencode "__RequestVerificationToken=$FORM_TOKEN" \
"https://$WEB_APP_HOST_NAME/")
rm -f "$COOKIE_JAR"
# The app redirects to its page once it stored the activity
if [[ $STATUS_CODE == 302 ]]; then
break
fi
if [ $ATTEMPT -lt $RETRY_COUNT ]; then
echo "Attempt $ATTEMPT of $RETRY_COUNT: the [$WEB_APP_NAME] web app answered the new activity with status code [$STATUS_CODE]. Waiting [$SLEEP] seconds before retry..."
sleep $SLEEP
fi
ATTEMPT=$((ATTEMPT + 1))
done
if [[ $STATUS_CODE == 302 ]]; then
echo "The [$WEB_APP_NAME] web app stored the [$ACTIVITY] activity"
else
echo "ERROR: the [$WEB_APP_NAME] web app answered the new activity with status code [$STATUS_CODE]"
EXIT_CODE=1
fi
# The requests the web app served (AppRequests)
query_workspace \
"Latest requests served by the [$WEB_APP_NAME] web app (AppRequests)" \
"AppRequests | project TimeGenerated, Name, ResultCode, DurationMs, Success | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Name:Name,Result:ResultCode,DurationMs:DurationMs,Success:Success}"
# The web app's calls to App Configuration, Key Vault and PostgreSQL (AppDependencies)
query_workspace \
"Latest dependencies of the [$WEB_APP_NAME] web app (AppDependencies)" \
"AppDependencies | project TimeGenerated, Name, Target, ResultCode, Success | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Name:Name,Target:Target,Result:ResultCode,Success:Success}"
# The application's own log records (AppTraces)
query_workspace \
"Latest log records of the [$WEB_APP_NAME] web app (AppTraces)" \
"AppTraces | project TimeGenerated, Message, SeverityLevel | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Message:Message,Severity:SeverityLevel}"
# The custom events of the activities added, including the one above, with the id of each activity (AppEvents)
query_workspace \
"Latest ActivityAdded custom events of the [$WEB_APP_NAME] web app (AppEvents)" \
"AppEvents | where Name == 'ActivityAdded' | project TimeGenerated, Name, ActivityId = tostring(Properties['activity.id']) | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Name:Name,Activity:ActivityId}"
# The custom metric that counts the activities by operation, exported once a minute (AppMetrics)
query_workspace \
"Custom metric of the activities (AppMetrics)" \
"AppMetrics | where Name == 'activities' | summarize Total = sum(Sum) by Name, Operation = tostring(Properties['operation'])" \
"[].{Metric:Name,Operation:Operation,Total:Total}"
# The exceptions the application logged with their stack trace (AppExceptions): a call to PostgreSQL that fails
# produces one
show_workspace_records \
"Latest exceptions of the [$WEB_APP_NAME] web app (AppExceptions)" \
"AppExceptions | project TimeGenerated, ExceptionType, OuterMessage | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Exception:ExceptionType,Message:OuterMessage}" \
"No exceptions: the [$WEB_APP_NAME] web app logged none."
exit $EXIT_CODE