From c84f14010c4999c837e29277876ac4efee9d635f Mon Sep 17 00:00:00 2001 From: Manda-supraja26 Date: Tue, 13 Feb 2024 18:47:09 +0530 Subject: [PATCH 1/2] Adding pgbench dashboard to grafonnet --- assets/pgbench-dashboard/annotation.libsonnet | 22 +++ assets/pgbench-dashboard/panels.libsonnet | 173 ++++++++++++++++++ assets/pgbench-dashboard/queries.libsonnet | 104 +++++++++++ assets/pgbench-dashboard/variables.libsonnet | 30 +++ .../General/pgbench-dashboard-v2.jsonnet | 32 ++++ 5 files changed, 361 insertions(+) create mode 100644 assets/pgbench-dashboard/annotation.libsonnet create mode 100644 assets/pgbench-dashboard/panels.libsonnet create mode 100644 assets/pgbench-dashboard/queries.libsonnet create mode 100644 assets/pgbench-dashboard/variables.libsonnet create mode 100644 templates/General/pgbench-dashboard-v2.jsonnet diff --git a/assets/pgbench-dashboard/annotation.libsonnet b/assets/pgbench-dashboard/annotation.libsonnet new file mode 100644 index 0000000..f61abd5 --- /dev/null +++ b/assets/pgbench-dashboard/annotation.libsonnet @@ -0,0 +1,22 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; +local annotation = g.dashboard.annotation; + +{ + run_start_timestamp: + annotation.withName('Run Start Time') + + annotation.withDatasource('$Datasource2') + + annotation.withEnable(true) + + annotation.withIconColor('#5794F2') + + annotation.withHide(false) + + annotation.target.withTags([]) + + annotation.withType('tags'), + + sample_start_timestamp: + annotation.withName('Sample Start Time') + + annotation.withDatasource('$Datasource2') + + annotation.withEnable(false) + + annotation.withIconColor('#B877D9') + + annotation.withHide(false) + + annotation.target.withTags([]) + + annotation.withType('tags'), +} \ No newline at end of file diff --git a/assets/pgbench-dashboard/panels.libsonnet b/assets/pgbench-dashboard/panels.libsonnet new file mode 100644 index 0000000..1175330 --- /dev/null +++ b/assets/pgbench-dashboard/panels.libsonnet @@ -0,0 +1,173 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +{ + timeSeries: { + local timeSeries = g.panel.timeSeries, + local custom = timeSeries.fieldConfig.defaults.custom, + local options = timeSeries.options, + + base(title, unit, targets, gridPos): + timeSeries.new(title) + + timeSeries.queryOptions.withTargets(targets) + + timeSeries.datasource.withType('elasticsearch') + + timeSeries.datasource.withUid('$Datasource1') + + timeSeries.standardOptions.withUnit(unit) + + timeSeries.gridPos.withX(gridPos.x) + + timeSeries.gridPos.withY(gridPos.y) + + timeSeries.gridPos.withH(gridPos.h) + + timeSeries.gridPos.withW(gridPos.w) + + custom.withDrawStyle("line") + + custom.withLineInterpolation("linear") + + custom.withBarAlignment(0) + + custom.withFillOpacity(10) + + custom.withGradientMode("none") + + custom.withSpanNulls(false) + + custom.withPointSize(5) + + custom.withSpanNulls(false) + + custom.stacking.withGroup("A") + + custom.stacking.withMode("none") + + custom.withShowPoints('never') + + timeSeries.queryOptions.withTimeFrom(null) + + timeSeries.queryOptions.withTimeShift(null) + + timeSeries.panelOptions.withTransparent(true), + + tps_report(title, unit, targets, gridPos): + self.base(title, unit, targets, gridPos) + + custom.withLineWidth(2) + + options.tooltip.withMode('multi') + + options.legend.withShowLegend(false) + + options.legend.withDisplayMode('list') + + options.legend.withPlacement('bottom'), + + + avg_tps(title, unit, targets, gridPos): + self.base(title, unit, targets, gridPos) + + options.legend.withShowLegend(true) + + options.legend.withDisplayMode('table') + + options.legend.withCalcs([ + "mean", + "max", + "min" + ]) + + options.legend.withPlacement('bottom') + + custom.withDrawStyle('bars') + + custom.withLineInterpolation('linear') + }, + + heatmap: { + local heatmap = g.panel.heatmap, + local custom = heatmap.fieldConfig.defaults.custom, + local options = heatmap.options, + + base(title, unit, targets, gridPos): + heatmap.new(title) + + heatmap.queryOptions.withTargets(targets) + + heatmap.datasource.withType('elasticsearch') + + heatmap.datasource.withUid('$Datasource1') + + heatmap.standardOptions.withUnit(unit) + + heatmap.gridPos.withX(gridPos.x) + + heatmap.gridPos.withY(gridPos.y) + + heatmap.gridPos.withH(gridPos.h) + + heatmap.gridPos.withW(gridPos.w) + + custom.scaleDistribution.withType('linear') + + custom.hideFrom.withLegend(false) + + custom.hideFrom.withTooltip(false) + + custom.hideFrom.withViz(false) + + options.withCalculate(true) + + options.yAxis.withAxisPlacement('left') + + options.yAxis.withReverse(false) + + options.yAxis.withUnit('ms') + + options.rowsFrame.withLayout('auto') + + options.color.HeatmapColorOptions.withMode('scheme') + + options.color.HeatmapColorOptions.withFill('dark-orange') + + options.color.HeatmapColorOptions.withScale('exponential') + + options.color.HeatmapColorOptions.withExponent(0.5) + + options.color.HeatmapColorOptions.withScheme('Oranges') + + options.color.HeatmapColorOptions.withSteps(128) + + options.color.HeatmapColorOptions.withReverse(false) + + options.withCellGap(2) + + options.filterValues.FilterValueRange.withLe(1e-9) + + options.tooltip.withShow(false) + + options.tooltip.withYHistogram(false) + + options.legend.withShow(true) + + options.exemplars.withColor('rgba(255,0,255,0.7)') + + options.withShowValue('never') + + heatmap.panelOptions.withTransparent(true) + }, + + table: { + local table = g.panel.table, + local custom = table.fieldConfig.defaults.custom, + local options = table.options, + + base(title, targets, gridPos): + table.new(title) + + table.queryOptions.withTargets(targets) + + table.datasource.withType('elasticsearch') + + table.datasource.withUid('$Datasource1') + + table.gridPos.withX(gridPos.x) + + table.gridPos.withY(gridPos.y) + + table.gridPos.withH(gridPos.h) + + table.gridPos.withW(gridPos.w) + + options.withShowHeader(true) + + options.footer.TableFooterOptions.withShow(false) + + options.footer.TableFooterOptions.withReducer('sum') + + options.footer.TableFooterOptions.withCountRows(false) + + custom.withAlign('auto') + + custom.withInspect(false) + + table.panelOptions.withTransparent(true) + + table.queryOptions.withTimeFrom(null) + + table.queryOptions.withTimeShift(null) + + table.standardOptions.color.withMode('thresholds') + + table.queryOptions.withTransformations([ + { + "id": "seriesToColumns", + "options": { + "reducers": [] + } + } + ]) + + table.standardOptions.withOverrides([ + { + "matcher": { + "id": "byName", + "options": "Average latency_ms" + }, + "properties": [ + { + "id": "displayName", + "value": "Avg latency" + }, + { + "id": "decimals", + "value": "2" + }, + { + "id": "custom.align", + "value": null + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Average tps" + }, + "properties": [ + { + "id": "displayName", + "value": "Avg TPS" + }, + { + "id": "decimals", + "value": "2" + }, + { + "id": "custom.align", + "value": null + } + ] + } + ]) + } +} \ No newline at end of file diff --git a/assets/pgbench-dashboard/queries.libsonnet b/assets/pgbench-dashboard/queries.libsonnet new file mode 100644 index 0000000..23d0a4f --- /dev/null +++ b/assets/pgbench-dashboard/queries.libsonnet @@ -0,0 +1,104 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; +local variables = import './variables.libsonnet'; +local elasticsearch = g.query.elasticsearch; + +{ + tps_report: { + query(): + elasticsearch.withAlias(null) + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("2") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField("tps") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum') + + elasticsearch.metrics.MetricAggregationWithSettings.CumulativeSum.withPipelineAgg('select metric') + + elasticsearch.metrics.MetricAggregationWithSettings.BucketScript.pipelineVariables.withName('var1') + + elasticsearch.metrics.MetricAggregationWithSettings.BucketScript.pipelineVariables.withPipelineAgg('select metric') + ]) + + elasticsearch.withQuery('(user = $user) AND (uuid = $uuid)') + + elasticsearch.withTimeField('timestamp') + }, + + avg_tps: { + query(): + elasticsearch.withAlias(null) + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("description.keyword") + + elasticsearch.bucketAggs.Terms.withId("6") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('asc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("4") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null) + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("tps_incl_con_est") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + + elasticsearch.metrics.MetricAggregationWithSettings.CumulativeSum.withPipelineAgg('select metric') + ]) + + elasticsearch.withQuery('(uuid.keyword=$uuid) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp'), + }, + + latency_report: { + query(): + elasticsearch.withAlias(null) + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("2") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null) + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("latency_ms") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + ]) + + elasticsearch.withQuery('(uuid.keyword=$uuid) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp'), + }, + + results: { + query(): + elasticsearch.withAlias(null) + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("user.keyword") + + elasticsearch.bucketAggs.Terms.withId("1") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("latency_ms") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("4") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg'), + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("tps") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("20") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + + ]) + + elasticsearch.withQuery('(uuid.keyword=$uuid) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp'), + } +} \ No newline at end of file diff --git a/assets/pgbench-dashboard/variables.libsonnet b/assets/pgbench-dashboard/variables.libsonnet new file mode 100644 index 0000000..e92165f --- /dev/null +++ b/assets/pgbench-dashboard/variables.libsonnet @@ -0,0 +1,30 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; +local var = g.dashboard.variable; + +{ + Datasource1: + var.datasource.new('Datasource1','elasticsearch') + + var.datasource.withRegex("") + + var.query.generalOptions.withLabel('pgbench-results datasource') + + var.query.withRefresh(1), + + Datasource2: + var.datasource.new('Datasource2','elasticsearch') + + var.datasource.withRegex("") + + var.query.generalOptions.withLabel('pgbench-summary datasource') + + var.query.withRefresh(1), + + uuid: + var.query.new('uuid','{"find": "terms", "field": "uuid.keyword"}') + + var.query.withDatasourceFromVariable(self.Datasource1) + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true) + + var.query.withRefresh(2), + + user: + var.query.new('user','{"find": "terms", "field": "user.keyword"}') + + var.query.withDatasourceFromVariable(self.Datasource1) + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true) + + var.query.withRefresh(2), +} \ No newline at end of file diff --git a/templates/General/pgbench-dashboard-v2.jsonnet b/templates/General/pgbench-dashboard-v2.jsonnet new file mode 100644 index 0000000..ccc0c45 --- /dev/null +++ b/templates/General/pgbench-dashboard-v2.jsonnet @@ -0,0 +1,32 @@ +local annotation = import '../../assets/pgbench-dashboard/annotation.libsonnet'; +local panels = import '../../assets/pgbench-dashboard/panels.libsonnet'; +local queries = import '../../assets/pgbench-dashboard/queries.libsonnet'; +local variables = import '../../assets/pgbench-dashboard/variables.libsonnet'; +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +g.dashboard.new('Pgbench') ++ g.dashboard.time.withFrom('now/y') ++ g.dashboard.time.withTo('now') ++ g.dashboard.withTimezone('utc') ++ g.dashboard.timepicker.withRefreshIntervals(['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d']) ++ g.dashboard.timepicker.withTimeOptions(['5m', '15m', '1h', '6h', '12h', '24h', '2d', '7d', '30d']) ++ g.dashboard.withRefresh('') ++ g.dashboard.withEditable(true) ++ g.dashboard.graphTooltip.withSharedCrosshair() ++ g.dashboard.withVariables([ + variables.Datasource1, + variables.Datasource2, + variables.uuid, + variables.user, +]) ++ g.dashboard.withAnnotations([ + annotation.run_start_timestamp, + annotation.sample_start_timestamp, +]) ++ g.dashboard.withPanels([ + panels.timeSeries.tps_report('TPS Report', 'ops', queries.tps_report.query(), { x: 0, y: 0, w: 12, h: 9 }), + panels.timeSeries.avg_tps('Overall Average TPS Per Run', 'ops', queries.avg_tps.query(), { x: 12, y: 0, w: 12, h: 9 }), + panels.heatmap.base('Latency Report', 'ms', queries.latency_report.query(), { x: 0, y: 9, w: 12, h: 9 }), + panels.table.base('Result Summary', queries.results.query(), { x: 12, y: 9, w: 12, h: 9 }), + +]) From 09f2070e1c1ec87a89ad1db1cf2d69787273d68e Mon Sep 17 00:00:00 2001 From: Manda-supraja26 Date: Fri, 16 Feb 2024 16:21:58 +0530 Subject: [PATCH 2/2] Updates in Pgbench dashboard --- assets/pgbench-dashboard/queries.libsonnet | 6 +++--- templates/General/pgbench-dashboard-v2.jsonnet | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/assets/pgbench-dashboard/queries.libsonnet b/assets/pgbench-dashboard/queries.libsonnet index 23d0a4f..aae5f3f 100644 --- a/assets/pgbench-dashboard/queries.libsonnet +++ b/assets/pgbench-dashboard/queries.libsonnet @@ -13,7 +13,7 @@ local elasticsearch = g.query.elasticsearch; + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") - + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null), + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0), ]) + elasticsearch.withMetrics([ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField("tps") @@ -44,7 +44,7 @@ local elasticsearch = g.query.elasticsearch; + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") - + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null) + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0) ]) + elasticsearch.withMetrics([ elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("tps_incl_con_est") @@ -66,7 +66,7 @@ local elasticsearch = g.query.elasticsearch; + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") - + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(null) + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0) ]) + elasticsearch.withMetrics([ elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("latency_ms") diff --git a/templates/General/pgbench-dashboard-v2.jsonnet b/templates/General/pgbench-dashboard-v2.jsonnet index ccc0c45..ef389db 100644 --- a/templates/General/pgbench-dashboard-v2.jsonnet +++ b/templates/General/pgbench-dashboard-v2.jsonnet @@ -28,5 +28,4 @@ g.dashboard.new('Pgbench') panels.timeSeries.avg_tps('Overall Average TPS Per Run', 'ops', queries.avg_tps.query(), { x: 12, y: 0, w: 12, h: 9 }), panels.heatmap.base('Latency Report', 'ms', queries.latency_report.query(), { x: 0, y: 9, w: 12, h: 9 }), panels.table.base('Result Summary', queries.results.query(), { x: 12, y: 9, w: 12, h: 9 }), - ])