Skip to content

Commit

Permalink
Add support for the float timestamps
Browse files Browse the repository at this point in the history
  • Loading branch information
lunaticusgreen committed Nov 24, 2024
1 parent aa7cb45 commit f905351
Show file tree
Hide file tree
Showing 11 changed files with 610 additions and 14 deletions.
23 changes: 23 additions & 0 deletions docker/clickhouse/init_schema.sql
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,26 @@ ORDER BY _time;
INSERT INTO default.test_barchart
SELECT now() - INTERVAL number HOUR AS _time, map ('rocketStage', concat('stage', toString(number % 4)), 'rp_prescreenStep', '(prescreen)') AS user_metadata_map, rand() % 100 AS alloc_cost, concat('cluster', toString(number%4)) AS cluster, multiIf(cluster='cluster2', 'RESOURCE_SLEEP', cluster='cluster3', 'INTERACTIVE', concat('RESOURCE', rand() % 4)) AS hpcod_resource_name
FROM numbers(24);

-- Drop the table if it exists
DROP TABLE IF EXISTS default.test_timestamp_formats SYNC;

-- Create the table with correct syntax
CREATE TABLE default.test_timestamp_formats (
t DateTime64(3),
tFloat Float64,
tDecimal Decimal64(3),
tUInt64 UInt64,
value UInt64
) ENGINE = MergeTree()
ORDER BY (t, tFloat, tDecimal, tUInt64);

-- Insert data into the table
INSERT INTO default.test_timestamp_formats
SELECT
now64(3) + INTERVAL number SECOND AS t,
toFloat64(now() + INTERVAL number SECOND) + randUniform(0, 1) AS tFloat,
toDecimal64(toFloat64(now() + INTERVAL number SECOND) + randUniform(0, 1), 3) AS tDecimal,
toUInt64(now() + INTERVAL number SECOND) + toUInt64(randUniform(0, 1000)) AS tUInt64,
toUInt64(rand() * 100) AS value
FROM numbers(86400);
307 changes: 307 additions & 0 deletions docker/grafana/dashboards/float_timestamps_support_issue_626.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,307 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 13,
"links": [],
"panels": [
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "Float timestamps without MS https://github.com/Altinity/clickhouse-grafana/issues/626",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.3.0+security-01",
"targets": [
{
"adHocFilters": [],
"add_metadata": true,
"contextWindowSize": 10,
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateColDataType": "",
"dateLoading": false,
"dateTimeColDataType": "tFloat",
"dateTimeType": "FLOAT",
"datetimeLoading": false,
"editorMode": "sql",
"extrapolate": true,
"format": "time_series",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "SELECT\n $timeSeries as t,\n count()\nFROM $table\n\nWHERE $timeFilter\nGROUP BY t\n\nORDER BY t",
"rawQuery": "/* grafana dashboard=Float timestamps issue 626, user=0 */\nSELECT\n round(too_big_value * 1000) as t,\n count()\nFROM default.test_timestamp_formats\n\nWHERE too_big_value >= 1732444676 AND too_big_value <= 1732445576\nGROUP BY t\n\nORDER BY t",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_timestamp_formats",
"tableLoading": false
}
],
"title": "Float Timestamps",
"type": "timeseries"
},
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "Float timestamps https://github.com/Altinity/clickhouse-grafana/issues/626",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.3.0+security-01",
"targets": [
{
"adHocFilters": [],
"add_metadata": true,
"contextWindowSize": 10,
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateColDataType": "",
"dateLoading": false,
"dateTimeColDataType": "tFloat",
"dateTimeType": "FLOAT",
"datetimeLoading": false,
"editorMode": "sql",
"extrapolate": true,
"format": "time_series",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"interval": "",
"intervalFactor": 1,
"query": "SELECT\n $timeSeriesMs as t,\n count()\nFROM $table\n\nWHERE $timeFilterMs\nGROUP BY t\n\nORDER BY t\n\n",
"rawQuery": "/* grafana dashboard=Float timestamps issue 626, user=0 */\nSELECT\n (intDiv(tFloat * 1000, 1000) * 1000) as t,\n count()\nFROM default.test_timestamp_formats\n\nWHERE tFloat >= toFloat64(1732444676627/1000) AND tFloat <= toFloat64(1732445576627/1000)\nGROUP BY t\n\nORDER BY t",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_timestamp_formats",
"tableLoading": false
}
],
"title": "Float Timestamps with MS",
"type": "timeseries"
}
],
"preload": false,
"refresh": "",
"schemaVersion": 40,
"tags": [],
"templating": {
"list": [
{
"baseFilters": [],
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"filters": [],
"name": "adhoc_variable",
"type": "adhoc"
},
{
"current": {
"text": "service_name",
"value": "service_name"
},
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"definition": "SELECT name FROM system.columns WHERE database='default' AND table='test_grafana' AND type ILIKE '%String%'",
"includeAll": false,
"name": "split",
"options": [],
"query": "SELECT name FROM system.columns WHERE database='default' AND table='test_grafana' AND type ILIKE '%String%'",
"refresh": 1,
"regex": "",
"type": "query"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Float timestamps issue 626",
"uid": "de4e7cr8tkdtse",
"version": 1,
"weekStart": ""
}
12 changes: 12 additions & 0 deletions pkg/eval_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -985,6 +985,9 @@ func (q *EvalQuery) getTimeSeries(dateTimeType string) string {
if dateTimeType == "DATETIME64" {
return "(intDiv(toFloat64($dateTimeCol) * 1000, ($interval * 1000)) * ($interval * 1000))"
}
if dateTimeType == "FLOAT" {
return "round($dateTimeCol * 1000)"
}
return "(intDiv($dateTimeCol, $interval) * $interval) * 1000"
}

Expand All @@ -995,6 +998,9 @@ func (q *EvalQuery) getTimeSeriesMs(dateTimeType string) string {
if dateTimeType == "DATETIME64" {
return "(intDiv(toFloat64($dateTimeCol) * 1000, $__interval_ms) * $__interval_ms)"
}
if dateTimeType == "FLOAT" {
return "(intDiv($dateTimeCol * 1000, $__interval_ms) * $__interval_ms)"
}
return "(intDiv($dateTimeCol, $__interval_ms) * $__interval_ms)"
}

Expand All @@ -1010,6 +1016,9 @@ func (q *EvalQuery) getDateTimeFilter(dateTimeType string) string {
if dateTimeType == "DATETIME64" {
return "toDateTime64(" + t + ", 3)"
}
if dateTimeType == "FLOAT" {
return t
}
return t
}
return "$dateTimeCol >= " + convertFn("$from") + " AND $dateTimeCol <= " + convertFn("$to")
Expand All @@ -1023,6 +1032,9 @@ func (q *EvalQuery) getDateTimeFilterMs(dateTimeType string) string {
if dateTimeType == "DATETIME64" {
return "toDateTime64(" + t + ", 3)"
}
if dateTimeType == "FLOAT" {
return "toFloat64(" + t + ")"
}
return t
}
return "$dateTimeCol >= " + convertFn("$__from/1000") + " AND $dateTimeCol <= " + convertFn("$__to/1000")
Expand Down
Loading

0 comments on commit f905351

Please sign in to comment.