Repository navigation
Expand file tree
/
Copy pathquery-telemetry.sh
More file actions
executable file
·181 lines (159 loc) · 7.04 KB
/
Copy pathquery-telemetry.sh
File metadata and controls
executable file
·181 lines (159 loc) · 7.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#!/bin/bash
# Usage: query-telemetry.sh
#
# Reads back what the Functions host and the Python worker of the capture processor sent through the connection string
# of the Application Insights resource to its Log Analytics workspace: the invocations of the CaptureProcessor function
# (AppRequests), its log records (AppTraces), the DeviceSummarized custom event of each device summary (AppEvents), the
# custom metric that counts the readings and the excursions it summarized (AppMetrics), and the exceptions it logged,
# if any (AppExceptions). Run it after run-pipeline.sh, which sends the readings that Capture archives.
#
# The records reach the workspace within seconds on the emulator. On Azure they take minutes, so every query is
# retried: raise RETRY_COUNT there.
# Variables (the same defaults and environment overrides as the deploy scripts)
PREFIX="${PREFIX:-local}"
SUFFIX="${SUFFIX:-test}"
RESOURCE_GROUP_NAME="${PREFIX}-ehgrid-rg"
LOG_ANALYTICS_NAME="${PREFIX}-ehgrid-log-analytics"
RETRY_COUNT="${RETRY_COUNT:-30}"
SLEEP="${SLEEP:-10}"
EXIT_CODE=0
# Run a KQL query on the Log Analytics workspace and print the records it returns as a table, retrying while the
# records are still on their way to the workspace
# $1: what the query reads
# $2: the KQL query
# $3: the JMESPath projection that names and orders the columns of the table
# $4: the number of records the query must return (1 when not given)
query_workspace() {
local description=$1
local query=$2
local columns=$3
local minimum_records=${4:-1}
local records
local attempt=1
echo -e "\n$description:\n"
while [ $attempt -le $RETRY_COUNT ]; do
records=$(az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "length(@)" \
--output tsv \
--only-show-errors)
if [[ $? == 0 && $records -ge $minimum_records ]]; then
az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "$columns" \
--output table \
--only-show-errors
if [[ $? != 0 ]]; then
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace did not answer the query that shows the records"
EXIT_CODE=1
fi
return 0
else
if [ $attempt -lt $RETRY_COUNT ]; then
echo "Attempt $attempt of $RETRY_COUNT: [${records:-0}] of the [$minimum_records] records expected so far. Waiting [$SLEEP] seconds before retry..."
sleep $SLEEP
fi
attempt=$((attempt + 1))
fi
done
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace returned [${records:-0}] of the [$minimum_records] records expected after $RETRY_COUNT attempts."
echo "On Azure the first records of a new component can take several minutes: run the script again later or raise RETRY_COUNT."
EXIT_CODE=1
}
# Run a KQL query on the Log Analytics workspace once and print the records it returns as a table, or say that there
# are none: for records that only an unexpected event produces
# $1: what the query reads
# $2: the KQL query
# $3: the JMESPath projection that names and orders the columns of the table
# $4: what it means when there are no records
show_workspace_records() {
local description=$1
local query=$2
local columns=$3
local when_empty=$4
local records
echo -e "\n$description:\n"
records=$(az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "length(@)" \
--output tsv \
--only-show-errors)
if [[ $? != 0 ]]; then
echo "ERROR: the [$LOG_ANALYTICS_NAME] Log Analytics workspace did not answer the query"
EXIT_CODE=1
elif [[ $records == 0 ]]; then
echo "$when_empty"
else
az monitor log-analytics query \
--workspace "$WORKSPACE_ID" \
--analytics-query "$query" \
--query "$columns" \
--output table \
--only-show-errors
fi
}
# Check whether the log-analytics extension of the Azure CLI, which provides the az monitor log-analytics query
# command, is installed
echo "Checking if the [log-analytics] extension of the Azure CLI is installed..."
az extension show --name log-analytics --only-show-errors &>/dev/null
if [[ $? != 0 ]]; then
echo "No [log-analytics] extension of the Azure CLI is actually installed"
echo "Installing the [log-analytics] extension of the Azure CLI..."
# Install the log-analytics extension
az extension add \
--name log-analytics \
--only-show-errors 1>/dev/null
if [[ $? == 0 ]]; then
echo "[log-analytics] extension of the Azure CLI successfully installed"
else
echo "Failed to install the [log-analytics] extension of the Azure CLI"
exit 1
fi
else
echo "[log-analytics] extension of the Azure CLI already installed"
fi
# Get the workspace id (customer id) of the Log Analytics workspace, which identifies it to the query API
echo "Retrieving the workspace id of the [$LOG_ANALYTICS_NAME] Log Analytics workspace..."
WORKSPACE_ID=$(az monitor log-analytics workspace show \
--resource-group "$RESOURCE_GROUP_NAME" \
--workspace-name "$LOG_ANALYTICS_NAME" \
--query customerId \
--output tsv \
--only-show-errors)
if [[ -n $WORKSPACE_ID ]]; then
echo "Workspace id [$WORKSPACE_ID] of the [$LOG_ANALYTICS_NAME] Log Analytics workspace successfully retrieved"
else
echo "Failed to retrieve the workspace id of the [$LOG_ANALYTICS_NAME] Log Analytics workspace"
exit 1
fi
# The invocations of the CaptureProcessor function, one per batch of Capture notifications (AppRequests)
query_workspace \
"Invocations of the function (AppRequests)" \
"AppRequests | summarize Invocations = count(), Failed = countif(Success == false) by Name | order by Name asc" \
"[].{Function:Name,Invocations:Invocations,Failed:Failed}"
# The log records of the function, such as CAPTURE_BATCH and DEVICE_SUMMARY (AppTraces)
query_workspace \
"Latest log records of the function (AppTraces)" \
"AppTraces | project TimeGenerated, Message, SeverityLevel | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Message:Message,Severity:SeverityLevel}"
# The DeviceSummarized custom events, counted per device (AppEvents)
query_workspace \
"DeviceSummarized custom events, per device (AppEvents)" \
"AppEvents | where Name == 'DeviceSummarized' | summarize Events = count(), Excursions = sum(toint(Properties['device.excursions'])) by Device = tostring(Properties['device.id']) | order by Device asc" \
"[].{Device:Device,Events:Events,Excursions:Excursions}"
# The custom metric that counts the readings and the excursions the function summarized, exported once a minute
# (AppMetrics)
query_workspace \
"Custom metric of the readings processed (AppMetrics)" \
"AppMetrics | where Name == 'readings_processed' | summarize Total = sum(Sum) by Name, Kind = tostring(Properties['kind']) | order by Kind asc" \
"[].{Metric:Name,Kind:Kind,Total:Total}"
# The exceptions the functions logged with their stack trace (AppExceptions)
show_workspace_records \
"Latest exceptions of the function (AppExceptions)" \
"AppExceptions | project TimeGenerated, ExceptionType, OuterMessage | order by TimeGenerated desc | take 10" \
"[].{Time:TimeGenerated,Exception:ExceptionType,Message:OuterMessage}" \
"No exceptions: the function logged none."
exit $EXIT_CODE