apps/postgresql: Receiver Enhancements (Metric Additions) (#804)

* test submodule update for postgres featuregate * add resource attribute flattening * re-add representative * update submodule and integration test to generate more metrics * clean up generated metadata.yaml * try integration tests again * add table attribute back to workload.googleapis.com/postgresql.blocks_read * label fixes * try to fix sles int test * try one more time * add to docs * update operations collector * don't modify service files * update postgres SLES 15 install * update postgres SLES 15 install to not look for libs * fix modification of postgresql.conf * update to use exercise * supply passwords to exercise * remove docs updates
GoogleCloudPlatform · Sep 15, 2022 · 52d2bd7 · 52d2bd7
1 parent 6d9f2db
commit 52d2bd7
Show file tree

Hide file tree

Showing 7 changed files with 173 additions and 19 deletions.
diff --git a/apps/postgresql.go b/apps/postgresql.go
@@ -73,6 +73,11 @@ func (r MetricsReceiverPostgresql) Pipelines() []otel.Pipeline {
 		},
 		Processors: []otel.Component{
 			otel.NormalizeSums(),
+			otel.TransformationMetrics(
+				otel.FlattenResourceAttribute("postgresql.database.name", "database"),
+				otel.FlattenResourceAttribute("postgresql.table.name", "table"),
+				otel.FlattenResourceAttribute("postgresql.index.name", "index"),
+			),
 			otel.MetricsTransform(
 				otel.AddPrefix("workload.googleapis.com"),
 			),

diff --git a/confgenerator/testdata/valid/linux/metrics-receiver_postgresql/golden_otel.yaml b/confgenerator/testdata/valid/linux/metrics-receiver_postgresql/golden_otel.yaml
@@ -407,7 +407,7 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
-  metricstransform/postgresql_postgresql_1:
+  metricstransform/postgresql_postgresql_2:
     transforms:
     - action: update
       include: ^(.*)$$
@@ -417,6 +417,12 @@ processors:
   resourcedetection/_global_0:
     detectors:
     - gce
+  transform/postgresql_postgresql_1:
+    metrics:
+      queries:
+      - set(attributes["database"], resource.attributes["postgresql.database.name"])
+      - set(attributes["table"], resource.attributes["postgresql.table.name"])
+      - set(attributes["index"], resource.attributes["postgresql.index.name"])
 receivers:
   hostmetrics/default__pipeline_hostmetrics:
     collection_interval: 60s
@@ -490,7 +496,8 @@ service:
       - googlecloud
       processors:
       - normalizesums/postgresql_postgresql_0
-      - metricstransform/postgresql_postgresql_1
+      - transform/postgresql_postgresql_1
+      - metricstransform/postgresql_postgresql_2
       - resourcedetection/_global_0
       receivers:
       - postgresql/postgresql_postgresql

diff --git a/confgenerator/testdata/valid/linux/metrics-receiver_postgresql_tls/golden_otel.yaml b/confgenerator/testdata/valid/linux/metrics-receiver_postgresql_tls/golden_otel.yaml
@@ -407,7 +407,7 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
-  metricstransform/postgresql_postgresql_1:
+  metricstransform/postgresql_postgresql_2:
     transforms:
     - action: update
       include: ^(.*)$$
@@ -417,6 +417,12 @@ processors:
   resourcedetection/_global_0:
     detectors:
     - gce
+  transform/postgresql_postgresql_1:
+    metrics:
+      queries:
+      - set(attributes["database"], resource.attributes["postgresql.database.name"])
+      - set(attributes["table"], resource.attributes["postgresql.table.name"])
+      - set(attributes["index"], resource.attributes["postgresql.index.name"])
 receivers:
   hostmetrics/default__pipeline_hostmetrics:
     collection_interval: 60s
@@ -492,7 +498,8 @@ service:
       - googlecloud
       processors:
       - normalizesums/postgresql_postgresql_0
-      - metricstransform/postgresql_postgresql_1
+      - transform/postgresql_postgresql_1
+      - metricstransform/postgresql_postgresql_2
       - resourcedetection/_global_0
       receivers:
       - postgresql/postgresql_postgresql

diff --git a/confgenerator/testdata/valid/linux/metrics-receiver_postgresql_tls_no_sni/golden_otel.yaml b/confgenerator/testdata/valid/linux/metrics-receiver_postgresql_tls_no_sni/golden_otel.yaml
@@ -407,7 +407,7 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
-  metricstransform/postgresql_postgresql_1:
+  metricstransform/postgresql_postgresql_2:
     transforms:
     - action: update
       include: ^(.*)$$
@@ -417,6 +417,12 @@ processors:
   resourcedetection/_global_0:
     detectors:
     - gce
+  transform/postgresql_postgresql_1:
+    metrics:
+      queries:
+      - set(attributes["database"], resource.attributes["postgresql.database.name"])
+      - set(attributes["table"], resource.attributes["postgresql.table.name"])
+      - set(attributes["index"], resource.attributes["postgresql.index.name"])
 receivers:
   hostmetrics/default__pipeline_hostmetrics:
     collection_interval: 60s
@@ -493,7 +499,8 @@ service:
       - googlecloud
       processors:
       - normalizesums/postgresql_postgresql_0
-      - metricstransform/postgresql_postgresql_1
+      - transform/postgresql_postgresql_1
+      - metricstransform/postgresql_postgresql_2
       - resourcedetection/_global_0
       receivers:
       - postgresql/postgresql_postgresql

diff --git a/...enerator/testdata/valid/linux/metrics-receiver_postgresql_tls_with_certs/golden_otel.yaml b/...enerator/testdata/valid/linux/metrics-receiver_postgresql_tls_with_certs/golden_otel.yaml
@@ -407,7 +407,7 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
-  metricstransform/postgresql_postgresql_1:
+  metricstransform/postgresql_postgresql_2:
     transforms:
     - action: update
       include: ^(.*)$$
@@ -417,6 +417,12 @@ processors:
   resourcedetection/_global_0:
     detectors:
     - gce
+  transform/postgresql_postgresql_1:
+    metrics:
+      queries:
+      - set(attributes["database"], resource.attributes["postgresql.database.name"])
+      - set(attributes["table"], resource.attributes["postgresql.table.name"])
+      - set(attributes["index"], resource.attributes["postgresql.index.name"])
 receivers:
   hostmetrics/default__pipeline_hostmetrics:
     collection_interval: 60s
@@ -496,7 +502,8 @@ service:
       - googlecloud
       processors:
       - normalizesums/postgresql_postgresql_0
-      - metricstransform/postgresql_postgresql_1
+      - transform/postgresql_postgresql_1
+      - metricstransform/postgresql_postgresql_2
       - resourcedetection/_global_0
       receivers:
       - postgresql/postgresql_postgresql

diff --git a/integration_test/third_party_apps_data/applications/postgresql/exercise b/integration_test/third_party_apps_data/applications/postgresql/exercise
@@ -0,0 +1,7 @@
+set -e
+
+sudo su postgres -c "PGPASSWORD=abc123 psql postgres -h localhost -c  \"CREATE TABLE test1 (id serial PRIMARY KEY);\""
+sudo su postgres -c "PGPASSWORD=abc123 psql postgres -h localhost -c \"CREATE TABLE test2 (id serial PRIMARY KEY);\""
+sudo su postgres -c "PGPASSWORD=abc123 psql postgres -h localhost -c \"CREATE INDEX otelindex ON test1(id);\""
+sudo su postgres -c "PGPASSWORD=abc123 psql postgres -h localhost -c \"CREATE INDEX otel2index ON test2(id);\""
+sudo su postgres -c "PGPASSWORD=abc123 psql postgres -h localhost -c \"INSERT INTO test2 (id) VALUES(67);\""
diff --git a/integration_test/third_party_apps_data/applications/postgresql/metadata.yaml b/integration_test/third_party_apps_data/applications/postgresql/metadata.yaml
@@ -31,31 +31,145 @@ minimum_supported_agent_version:
 supported_operating_systems: linux
 supported_app_version: ["10.18+"]
 expected_metrics:
-  - type: workload.googleapis.com/postgresql.backends
+  - kind: GAUGE
+    labels:
+      database: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.backends
+    value_type: INT64
+    representative: true
+  - kind: CUMULATIVE
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.bgwriter.buffers.allocated
+    value_type: INT64
+  - kind: CUMULATIVE
+    labels:
+      source: backend|backend_fsync|checkpoints|bgwriter
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.bgwriter.buffers.writes
+    value_type: INT64
+  - kind: CUMULATIVE
+    labels:
+      type: requested|scheduled
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.bgwriter.checkpoint.count
+    value_type: INT64
+  - kind: CUMULATIVE
+    labels:
+      type: sync|write
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.bgwriter.duration
     value_type: INT64
-    kind: GAUGE
+  - kind: CUMULATIVE
     monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.bgwriter.maxwritten
+    value_type: INT64
+  - kind: CUMULATIVE
     labels:
       database: .*
-    representative: true
-  - type: workload.googleapis.com/postgresql.commits
+      table: .*
+      source: heap_read|heap_hit|idx_read|idx_hit|toast_read|toast_hit|tidx_read|tidx_hit
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.blocks_read
+    value_type: INT64
+  - kind: CUMULATIVE
+    labels:
+      database: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.commits
     value_type: INT64
-    kind: CUMULATIVE
+  - kind: GAUGE
     monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.connection.max
+    value_type: INT64
+  - kind: GAUGE
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.database.count
+    value_type: INT64
+  - kind: GAUGE
     labels:
       database: .*
-  - type: workload.googleapis.com/postgresql.db_size
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.db_size
     value_type: INT64
-    kind: GAUGE
+  - kind: CUMULATIVE
+    labels:
+      database: .*
+      table: .*
+      index: .*
     monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.index.scans
+    value_type: INT64
+  - kind: GAUGE
     labels:
       database: .*
-  - type: workload.googleapis.com/postgresql.rollbacks
+      table: .*
+      index: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.index.size
     value_type: INT64
-    kind: CUMULATIVE
+  - kind: CUMULATIVE
+    labels:
+      database: .*
+      table: .*
+      operation: ins|upd|del|hot_upd
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.operations
+    value_type: INT64
+  - kind: GAUGE
+    labels:
+      replication_client: .*
     monitored_resource: gce_instance
+    optional: true
+    type: workload.googleapis.com/postgresql.replication.data_delay
+    value_type: INT64
+  - kind: CUMULATIVE
     labels:
       database: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.rollbacks
+    value_type: INT64
+  - kind: GAUGE
+    labels:
+      database: .*
+      state: dead|live
+      table: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.rows
+    value_type: INT64
+  - kind: GAUGE
+    labels:
+      database: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.table.count
+    value_type: INT64
+  - kind: GAUGE
+    labels:
+      database: .*
+      table: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.table.size
+    value_type: INT64
+  - kind: CUMULATIVE
+    labels:
+      database: .*
+      table: .*
+    monitored_resource: gce_instance
+    type: workload.googleapis.com/postgresql.table.vacuum.count
+    value_type: INT64
+  - kind: GAUGE
+    monitored_resource: gce_instance
+    optional: true
+    type: workload.googleapis.com/postgresql.wal.age
+    value_type: INT64
+  - kind: GAUGE
+    labels:
+      operation: flush|replay|write
+      replication_client: .*
+    monitored_resource: gce_instance
+    optional: true
+    type: workload.googleapis.com/postgresql.wal.lag
+    value_type: INT64
 expected_logs:
   - log_name: postgresql_general
     fields:
@@ -80,7 +194,7 @@ expected_logs:
         description: Authenticated user for the action being logged when relevant
       - name: severity
         type: string
-        description: ''
+        description: ""
 configuration_options:
   logs:
     - type: postgresql_general
@@ -89,7 +203,7 @@ configuration_options:
           default: null
           description: The value must be postgresql_general.
         - name: include_paths
-          default: '[/var/log/postgresql/postgresql*.log, /var/lib/pgsql/data/log/postgresql*.log, /var/lib/pgsql/*/data/log/postgresql*.log]'
+          default: "[/var/log/postgresql/postgresql*.log, /var/lib/pgsql/data/log/postgresql*.log, /var/lib/pgsql/*/data/log/postgresql*.log]"
           description: A list of filesystem paths to read by tailing each file. A wild card (*) can be used in the paths.
         - name: exclude_paths
           default: null