Skip to content

Commit beaa6f1

Browse files
n1ru4ln1ru4l
committed
otel database migrations
1 parent 6b439ab commit beaa6f1

File tree

4 files changed

+166
-2
lines changed

4 files changed

+166
-2
lines changed

‎docker/configs/otel-collector/config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ exporters:
3737
async_insert: true
3838
username: test
3939
password: test
40-
create_schema: true
40+
create_schema: false
4141
ttl: 720h
4242
compress: lz4
4343
logs_table_name: otel_logs

‎docker/docker-compose.dev.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ services:
183183
condition: service_healthy
184184
build:
185185
context: ./configs/otel-collector
186-
dockerfile: ../otel-collector.dockerfile
186+
dockerfile: ./../../otel-collector.dockerfile
187187
environment:
188188
HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth'
189189
volumes:

‎packages/migrations/src/clickhouse-actions/015-otel-trace.ts

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
import type { Action } from '../clickhouse';
2+
3+
export const action: Action = async exec => {
4+
// Base tables as created by otel-exporter clickhouse
5+
await exec(`
6+
CREATE TABLE IF NOT EXISTS "otel_traces" (
7+
"Timestamp" DateTime64(9) CODEC(Delta(8), ZSTD(1))
8+
, "TraceId" String CODEC(ZSTD(1))
9+
, "SpanId" String CODEC(ZSTD(1))
10+
, "ParentSpanId" String CODEC(ZSTD(1))
11+
, "TraceState" String CODEC(ZSTD(1))
12+
, "SpanName" LowCardinality(String) CODEC(ZSTD(1))
13+
, "SpanKind" LowCardinality(String) CODEC(ZSTD(1))
14+
, "ServiceName" LowCardinality(String) CODEC(ZSTD(1))
15+
, "ResourceAttributes" Map(LowCardinality(String), String) CODEC(ZSTD(1))
16+
, "ScopeName" String CODEC(ZSTD(1))
17+
, "ScopeVersion" String CODEC(ZSTD(1))
18+
, "SpanAttributes" Map(LowCardinality(String), String) CODEC(ZSTD(1))
19+
, "Duration" UInt64 CODEC(ZSTD(1))
20+
, "StatusCode" LowCardinality(String) CODEC(ZSTD(1))
21+
, "StatusMessage" String CODEC(ZSTD(1))
22+
, "Events.Timestamp" Array(DateTime64(9)) CODEC(ZSTD(1))
23+
, "Events.Name" Array(LowCardinality(String)) CODEC(ZSTD(1))
24+
, "Events.Attributes" Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1))
25+
, "Links.TraceId" Array(String) CODEC(ZSTD(1))
26+
, "Links.SpanId" Array(String) CODEC(ZSTD(1))
27+
, "Links.TraceState" Array(String) CODEC(ZSTD(1))
28+
, "Links.Attributes" Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1))
29+
, INDEX "idx_trace_id" "TraceId" TYPE bloom_filter(0.001) GRANULARITY 1
30+
, INDEX "idx_res_attr_key" mapKeys("ResourceAttributes") TYPE bloom_filter(0.01) GRANULARITY 1
31+
, INDEX "idx_res_attr_value" mapValues("ResourceAttributes") TYPE bloom_filter(0.01) GRANULARITY 1
32+
, INDEX "idx_span_attr_key" mapKeys("SpanAttributes") TYPE bloom_filter(0.01) GRANULARITY 1
33+
, INDEX "idx_span_attr_value" mapValues("SpanAttributes") TYPE bloom_filter(0.01) GRANULARITY 1
34+
, INDEX "idx_duration" Duration TYPE minmax GRANULARITY 1
35+
)
36+
ENGINE = MergeTree
37+
PARTITION BY toDate("Timestamp")
38+
ORDER BY (
39+
"ServiceName"
40+
, "SpanName"
41+
, toDateTime("Timestamp")
42+
)
43+
TTL toDate("Timestamp") + toIntervalDay(365)
44+
SETTINGS
45+
index_granularity = 8192
46+
, ttl_only_drop_parts = 1
47+
`);
48+
49+
await exec(`
50+
CREATE TABLE IF NOT EXISTS "otel_traces_trace_id_ts" (
51+
"TraceId" String CODEC(ZSTD(1))
52+
, "Start" DateTime CODEC(Delta(4), ZSTD(1))
53+
, "End" DateTime CODEC(Delta(4), ZSTD(1))
54+
, INDEX "idx_trace_id" "TraceId" TYPE bloom_filter(0.01) GRANULARITY 1
55+
)
56+
ENGINE = MergeTree
57+
PARTITION BY toDate(Start)
58+
ORDER BY (
59+
"TraceId"
60+
, "Start"
61+
)
62+
TTL toDate("Start") + toIntervalDay(365)
63+
SETTINGS
64+
index_granularity = 8192
65+
, ttl_only_drop_parts = 1
66+
`);
67+
68+
await exec(`
69+
CREATE MATERIALIZED VIEW IF NOT EXISTS "otel_traces_trace_id_ts_mv" TO "otel_traces_trace_id_ts" (
70+
"TraceId" String
71+
, "Start" DateTime64(9)
72+
, "End" DateTime64(9)
73+
)
74+
AS (
75+
SELECT
76+
"TraceId"
77+
, min("Timestamp") AS "Start"
78+
, max("Timestamp") AS "End"
79+
FROM
80+
"otel_traces"
81+
WHERE
82+
"TraceId" != ''
83+
GROUP BY
84+
"TraceId"
85+
)
86+
`);
87+
88+
// Our own custom materialized views based on the base tables
89+
await exec(`
90+
CREATE TABLE IF NOT EXISTS "otel_traces_normalized" (
91+
"target_id" LowCardinality(String) CODEC(ZSTD(1))
92+
, "trace_id" String CODEC(ZSTD(1))
93+
, "span_id" String CODEC(ZSTD(1))
94+
, "timestamp" DateTime('UTC') CODEC(DoubleDelta, LZ4)
95+
, "operation_name" String CODEC(ZSTD(1))
96+
, "operation_type" LowCardinality(String) CODEC(ZSTD(1))
97+
, "operation_body" String CODEC(ZSTD(1))
98+
, "duration" UInt64 CODEC(T64, ZSTD(1))
99+
, "subgraph_names" Array(LowCardinality(String)) CODEC(ZSTD(1))
100+
, "http_status_code" String CODEC(ZSTD(1))
101+
, "http_method" String CODEC(ZSTD(1))
102+
, "http_host" String CODEC(ZSTD(1))
103+
, "http_route" String CODEC(ZSTD(1))
104+
, "http_url" String CODEC(ZSTD(1))
105+
, INDEX "idx_operation_name" "operation_name" TYPE bloom_filter(0.01) GRANULARITY 1
106+
, INDEX "idx_operation_type" "operation_type" TYPE bloom_filter(0.01) GRANULARITY 1
107+
, INDEX "idx_duration" "duration" TYPE minmax GRANULARITY 1
108+
, INDEX "idx_subgraph_names" "subgraph_names" TYPE bloom_filter(0.01) GRANULARITY 1
109+
, INDEX "idx_http_status_code" "http_status_code" TYPE bloom_filter(0.01) GRANULARITY 1
110+
, INDEX "idx_http_method" "http_method" TYPE bloom_filter(0.01) GRANULARITY 1
111+
, INDEX "idx_http_host" "http_host" TYPE bloom_filter(0.01) GRANULARITY 1
112+
, INDEX "idx_http_route" "http_route" TYPE bloom_filter(0.01) GRANULARITY 1
113+
, INDEX "idx_http_url" "http_url" TYPE bloom_filter(0.01) GRANULARITY 1
114+
)
115+
ENGINE = MergeTree
116+
PARTITION BY toDate(timestamp)
117+
ORDER BY ("target_id", "timestamp")
118+
TTL toDateTime(timestamp) + toIntervalDay(365)
119+
SETTINGS
120+
index_granularity = 8192
121+
, ttl_only_drop_parts = 1
122+
`);
123+
124+
await exec(`
125+
CREATE MATERIALIZED VIEW IF NOT EXISTS "otel_traces_normalized_mv" TO "otel_traces_normalized" (
126+
"trace_id" String
127+
, "span_id" String
128+
, "timestamp" DateTime('UTC')
129+
, "operation_name" String
130+
, "operation_type" LowCardinality(String)
131+
, "target_id" LowCardinality(String)
132+
, "duration" UInt64
133+
, "operation_body" String
134+
, "subgraph_names" Array(String)
135+
, "http_status_code" String
136+
, "http_host" String
137+
, "http_method" String
138+
, "http_route" String
139+
, "http_url" String
140+
)
141+
AS (
142+
SELECT
143+
"TraceId" as "trace_id"
144+
, "SpanId" AS "span_id"
145+
, toDateTime("Timestamp", 'UTC') AS "timestamp"
146+
, "SpanAttributes"['graphql.operation.name'] AS operation_name
147+
, toLowCardinality("SpanAttributes"['graphql.operation.type']) AS "operation_type"
148+
, toLowCardinality("SpanAttributes"['hive.target_id']) AS "target_id"
149+
, "Duration" AS "duration"
150+
, "SpanAttributes"['graphql.operation.document'] AS "operation_body"
151+
, arrayMap(x -> toLowCardinality(x), splitByChar(',', "SpanAttributes"['subgraph.names'])) AS "subgraph_names"
152+
, "SpanAttributes"['http.status_code'] AS "http_status_code"
153+
, "SpanAttributes"['http.host'] AS "http_host"
154+
, "SpanAttributes"['http.method'] AS "http_method"
155+
, "SpanAttributes"['http.route'] AS "http_route"
156+
, "SpanAttributes"['http.url'] AS "http_url"
157+
FROM
158+
"otel_traces"
159+
WHERE
160+
empty("ParentSpanId")
161+
)
162+
`);
163+
};

‎packages/migrations/src/clickhouse.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ export async function migrateClickHouse(
175175
import('./clickhouse-actions/012-coordinates-typename-index'),
176176
import('./clickhouse-actions/013-apply-ttl'),
177177
import('./clickhouse-actions/014-audit-logs-access-token'),
178+
import('./clickhouse-actions/015-otel-trace'),
178179
]);
179180

180181
async function actionRunner(action: Action, index: number) {

0 commit comments

Comments
 (0)