Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Display OVN metrics from worker node
Browse files Browse the repository at this point in the history
Metric containerCPU-AggregatedWorkers is NOT using the pod while
fetching the data, so none of the existing panels in the dashboard
are fetching OVN data from worker nodes as they use the pod in the
metric's query i.e
labels.pod.keyword: /ovnkube-node.*/

OVN IC is the default from OCP 4.14. So from 4.14 onwards,
ovnkube-master pod doesn't exist.
OVN now has 2 pods - ovnkube-control-plane & ovnkube-node.
ovnkube-control-plane pod exists on the master nodes. This pod contains
ovnkube-cluster-manager container with very minimal functionality and
we can ignore metrics for this pod and the container.
ovnkube-node pod exists on master, infra and worker nodes. This pod
contains all OVN containers (both core OVN and OVNK). We are interested
in metrics from the containers in this pod. So this patch creates
separate panels for master, worker and infra nodes to show these
containers CPU & memory usage.

Note: We are using the Average instead of the Sum across all panels
to ensure consistency with the worker node panel

Signed-off-by: venkataanil <[email protected]>
venkataanil committed Sep 10, 2024
1 parent eb3a042 commit 6c000c5
Showing 2 changed files with 11 additions and 191 deletions.
184 changes: 5 additions & 179 deletions assets/kube-burner-report-ocp-wrapper/queries.libsonnet
Original file line number Diff line number Diff line change
@@ -552,181 +552,7 @@ local elasticsearch = g.query.elasticsearch;
+ elasticsearch.withQuery('uuid.keyword: $uuid AND metricName.keyword: alert')
+ elasticsearch.withTimeField('timestamp'),
},
ovnKubeMasterPodStats: {
base(alias, query):
elasticsearch.withAlias(alias)
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.Terms.withField('labels.pod.keyword')
+ elasticsearch.bucketAggs.Terms.withId('3')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('0'),
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum'),
])
+ elasticsearch.withQuery(query)
+ elasticsearch.withTimeField('timestamp'),
queries(metric): [
self.base('{{labels.pod.keyword}}', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-master.*/'),
self.base('{{labels.pod.keyword}}', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-control-plane.*/'),
],
},
ovnKubeMasterStats: {
base(alias, query):
elasticsearch.withAlias(alias)
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.Terms.withField('labels.pod.keyword')
+ elasticsearch.bucketAggs.Terms.withId('4')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('0'),
elasticsearch.bucketAggs.DateHistogram.withField('labels.container.keyword')
+ elasticsearch.bucketAggs.Terms.withId('3')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('0'),
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Average.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg'),
])
+ elasticsearch.withQuery(query)
+ elasticsearch.withTimeField('timestamp'),
queries(metric): [
self.base('{{labels.pod.keyword}}-{{labels.container.keyword}}', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-master.*/'),
self.base('{{labels.pod.keyword}}-{{labels.container.keyword}}', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-control-plane.*/'),
],
},
ovnKubeNodePodStats: {
base(alias, query):
elasticsearch.withAlias(alias)
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.Terms.withField('labels.pod.keyword')
+ elasticsearch.bucketAggs.Terms.withId('3')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('1')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('5'),
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum'),
])
+ elasticsearch.withQuery(query)
+ elasticsearch.withTimeField('timestamp'),
queries(metric): [
self.base('{{labels.pod.keyword}}', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.namespace.keyword: "openshift-ovn-kubernetes" AND labels.pod.keyword: /ovnkube-node.*/'),
elasticsearch.withAlias('Aggregated')
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum'),
])
+ elasticsearch.withQuery('uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.namespace.keyword: "openshift-ovn-kubernetes" AND labels.pod.keyword: /ovnkube-node.*/')
+ elasticsearch.withTimeField('timestamp'),
],
},
ovnControllerStats: {
query(metric):
elasticsearch.withAlias('{{labels.pod.keyword}}')
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.Terms.withField('labels.pod.keyword')
+ elasticsearch.bucketAggs.Terms.withId('3')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('1')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('5'),
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Average.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg'),
])
+ elasticsearch.withQuery('uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.namespace.keyword: "openshift-ovn-kubernetes" AND labels.pod.keyword: /ovnkube-node.*/ AND labels.container.keyword: "ovn-controller"')
+ elasticsearch.withTimeField('timestamp'),
},
aggregatedOVNKubeMasterStats: {
base(alias, query):
elasticsearch.withAlias(alias)
+ elasticsearch.withBucketAggs([
elasticsearch.bucketAggs.Terms.withField('labels.container.keyword')
+ elasticsearch.bucketAggs.Terms.withId('3')
+ elasticsearch.bucketAggs.Terms.withType('terms')
+ elasticsearch.bucketAggs.Terms.settings.withOrder('desc')
+ elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term')
+ elasticsearch.bucketAggs.Terms.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.Terms.settings.withSize('0'),
elasticsearch.bucketAggs.DateHistogram.withField('timestamp')
+ elasticsearch.bucketAggs.DateHistogram.withId('2')
+ elasticsearch.bucketAggs.DateHistogram.withType('date_histogram')
+ elasticsearch.bucketAggs.DateHistogram.settings.withInterval('10s')
+ elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount('1')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone('utc')
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum'),
])
+ elasticsearch.withQuery(query)
+ elasticsearch.withTimeField('timestamp'),
queries(metric): [
self.base('', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-master.*/'),
self.base('', 'uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.pod.keyword: /ovnkube-control-plane.*/'),
],
},
aggregatedOVNKubeNodeStats: {
aggregatedOVNNodeStats: {
query(metric):
elasticsearch.withAlias('')
+ elasticsearch.withBucketAggs([
@@ -746,11 +572,11 @@ local elasticsearch = g.query.elasticsearch;
+ elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges('0'),
])
+ elasticsearch.withMetrics([
elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum'),
elasticsearch.metrics.MetricAggregationWithSettings.Average.withField('value')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withId('1')
+ elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('average'),
])
+ elasticsearch.withQuery('uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.namespace.keyword: "openshift-ovn-kubernetes" AND labels.pod.keyword: /ovnkube-node.*/')
+ elasticsearch.withQuery('uuid.keyword: $uuid AND metricName: "' + metric + '" AND labels.namespace.keyword: "openshift-ovn-kubernetes"')
+ elasticsearch.withTimeField('timestamp'),
},
etcd99thLatencies: {
18 changes: 6 additions & 12 deletions templates/CPT/kube-burner-report-ocp-wrapper.jsonnet
Original file line number Diff line number Diff line change
@@ -63,18 +63,12 @@ g.dashboard.new('Kube-burner Report - OCP wrapper')
+ g.panel.row.withGridPos({ x: 0, y: 14, w: 24, h: 1 })
+ g.panel.row.withCollapsed(true)
+ g.panel.row.withPanels([
panels.timeSeries.sortByMean('ovnkube-master pods CPU usage', 'percent', queries.ovnKubeMasterPodStats.queries('containerCPU'), { x: 0, y: 16, w: 12, h: 9 }, null),
panels.timeSeries.sortByMax('ovnkube-master pods Memory usage', 'bytes', queries.ovnKubeMasterPodStats.queries('containerMemory'), { x: 12, y: 16, w: 12, h: 9 }, null),
panels.timeSeries.sortByMean('ovnkube-master CPU usage', 'percent', queries.ovnKubeMasterStats.queries('containerCPU'), { x: 0, y: 25, w: 12, h: 8 }, null),
panels.timeSeries.sortByMaxCommon('ovnkube-master Memory Usage', 'bytes', queries.ovnKubeMasterStats.queries('containerMemory'), { x: 12, y: 25, w: 12, h: 8 }, null),
panels.timeSeries.sortByMean('ovnkube-node pods CPU Usage', 'percent', queries.ovnKubeNodePodStats.queries('containerCPU'), { x: 0, y: 33, w: 12, h: 8 }, null),
panels.timeSeries.sortByMean('ovnkube-node pods Memory Usage', 'bytes', queries.ovnKubeNodePodStats.queries('containerMemory'), { x: 12, y: 33, w: 12, h: 8 }, null),
panels.timeSeries.sortByMax('ovn-controller CPU Usage', 'percent', queries.ovnControllerStats.query('containerCPU'), { x: 0, y: 41, w: 12, h: 8 }, null),
panels.timeSeries.sortByMax('ovn-controller Memory Usage', 'bytes', queries.ovnControllerStats.query('containerMemory'), { x: 12, y: 41, w: 12, h: 8 }, null),
panels.timeSeries.withMeanMax('Aggregated OVNKube-master containers CPU', 'percent', queries.aggregatedOVNKubeMasterStats.queries('containerCPU'), { x: 0, y: 49, w: 12, h: 14 }, null),
panels.timeSeries.withMeanMax('Aggregated OVNKube-master containers memory', 'bytes', queries.aggregatedOVNKubeMasterStats.queries('containerMemory'), { x: 12, y: 49, w: 12, h: 14 }, null),
panels.timeSeries.withMeanMax('Aggregated OVNKube-node containers CPU', 'percent', queries.aggregatedOVNKubeNodeStats.query('containerCPU'), { x: 0, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.sortByMeanCommon('Aggregated OVNKube-node containers Memory', 'bytes', queries.aggregatedOVNKubeNodeStats.query('containerMemory'), { x: 12, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.withMeanMax('OVN containers CPU on Worker node', 'percent', queries.aggregatedOVNNodeStats.query('containerCPU-AggregatedWorkers'), { x: 0, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.sortByMeanCommon('OVN containers Memory on Worker node', 'bytes', queries.aggregatedOVNNodeStats.query('containerMemory-AggregatedWorkers'), { x: 12, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.withMeanMax('OVN containers CPU on Master node', 'percent', queries.aggregatedOVNNodeStats.query('containerCPU-Masters'), { x: 0, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.sortByMeanCommon('OVN containers Memory on Master node', 'bytes', queries.aggregatedOVNNodeStats.query('containerMemory-Masters'), { x: 12, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.withMeanMax('OVN containers CPU on Infra node', 'percent', queries.aggregatedOVNNodeStats.query('containerCPU-Infra'), { x: 0, y: 63, w: 12, h: 14 }, null),
panels.timeSeries.sortByMeanCommon('OVN containers Memory on Infra node', 'bytes', queries.aggregatedOVNNodeStats.query('containerMemory-Infra'), { x: 12, y: 63, w: 12, h: 14 }, null),
]),
g.panel.row.new('etcd')
+ g.panel.row.withGridPos({ x: 0, y: 14, w: 24, h: 1 })

0 comments on commit 6c000c5

Please sign in to comment.