doc2statsdmapper: run against zuul 11.1.0, add doc

Update missing zuul metrics since bumping to 11.1.0. Added documentation on how to properly update containers. Change-Id: Ia761bd2a1a015b18985909cd3ba441d0298125a0
softwarefactory-project · Nov 25, 2024 · fa168f9 · fa168f9
1 parent f96496a
commit fa168f9
Show file tree

Hide file tree

Showing 7 changed files with 176 additions and 35 deletions.
diff --git a/controllers/static/nodepool/statsd_mapping.yaml.tmpl b/controllers/static/nodepool/statsd_mapping.yaml.tmpl
@@ -1,5 +1,5 @@
 # Auto-generated with zuuldoc2statsdmapper.py, please check with: 
-# podman run --rm -v controllers/static/nodepool/statsd_mapping.yaml:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml
+# podman run --rm -v controllers/static/nodepool/statsd_mapping.yaml.tmpl:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml
 #
 mappings:
 - help: Description at https://zuul-ci.org/docs/nodepool/latest/operation.html#stat-nodepool.dib_image_build.<diskimage_name>.<ext>.size
@@ -200,15 +200,22 @@ mappings:
     hostname: "$1"
   match: nodepool.launcher.*.zk.image_cache.playback_queue
   name: nodepool_launcher_zk_image_cache_playback_queue
-{{- range . }}
-- name: {{ .Name }}
-  match: {{ .Match }}
-  help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics
-  labels:
-{{- range .Labels }}
-    {{ .LabelName }}: "{{ .LabelValue -}}"
-{{- end }}
-{{- end }}
+
+# Uncomment below after testing
+# {{- range . }}
+# - name: {{ .Name }}
+#   match: {{ .Match }}
+#   help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics
+#   labels:
+# {{- range .Labels }}
+#     {{ .LabelName }}: "{{ .LabelValue -}}"
+# {{- end }}
+# {{- end }}
+# - action: drop
+#   match: .
+#   match_type: regex
+#   name: "dropped"
+
 - action: drop
   match: .
   match_type: regex

diff --git a/controllers/static/zuul/statsd_mapping.yaml b/controllers/static/zuul/statsd_mapping.yaml
@@ -179,46 +179,56 @@ mappings:
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue
   name: zuul_tenant_pipeline_queue
-- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue
-    name>
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>
   labels:
     pipeline: "$2"
     queue: "$3"
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue.*
   name: zuul_tenant_pipeline_queue
-- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue
-    name>.current_changes
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.current_changes
   labels:
     pipeline: "$2"
     queue: "$3"
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue.*.current_changes
   name: zuul_tenant_pipeline_queue_current_changes
-- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue
-    name>.window
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.window
   labels:
     pipeline: "$2"
     queue: "$3"
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue.*.window
   name: zuul_tenant_pipeline_queue_window
-- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue
-    name>.resident_time
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.resident_time
   labels:
     pipeline: "$2"
     queue: "$3"
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue.*.resident_time
   name: zuul_tenant_pipeline_queue_resident_time
-- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue
-    name>.total_changes
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.total_changes
   labels:
     pipeline: "$2"
     queue: "$3"
     tenant: "$1"
   match: zuul.tenant.*.pipeline.*.queue.*.total_changes
   name: zuul_tenant_pipeline_queue_total_changes
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch
+  labels:
+    pipeline: "$2"
+    queue: "$3"
+    tenant: "$1"
+  match: zuul.tenant.*.pipeline.*.queue.*.branch
+  name: zuul_tenant_pipeline_queue_branch
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.queue.<queue>.branch.<branch>
+  labels:
+    branch: "$4"
+    pipeline: "$2"
+    queue: "$3"
+    tenant: "$1"
+  match: zuul.tenant.*.pipeline.*.queue.*.branch.*
+  name: zuul_tenant_pipeline_queue_branch
 - help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant.<tenant>.pipeline.<pipeline>.project
   labels:
     pipeline: "$2"
@@ -464,6 +474,11 @@ mappings:
     executor: "$1"
   match: zuul.executor.*.pct_used_hdd
   name: zuul_executor_pct_used_hdd
+- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.executor.<executor>.pct_used_inodes
+  labels:
+    executor: "$1"
+  match: zuul.executor.*.pct_used_inodes
+  name: zuul_executor_pct_used_inodes
 - help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.executor.<executor>.pct_used_ram
   labels:
     executor: "$1"
@@ -659,6 +674,7 @@ mappings:
     hostname: "$1"
   match: zuul.web.server.*.streamers
   name: zuul_web_server_streamers
+
 - action: drop
   match: .
   match_type: regex

diff --git a/doc/developer/images.md b/doc/developer/images.md
@@ -11,6 +11,7 @@ for development purposes.
 1. [Modify an existing image](#modify-an-existing-image)
 1. [Create and use an image from a Containerfile](#create-and-use-an-image-from-a-containerfile)
 1. [Edit Zuul source code and mount in a pod](#edit-the-zuul-source-code-and-mount-in-a-pod)
+1. [Upgrading an image](#upgrading-an-image)
 
 ## Root access inside containers
 
@@ -134,3 +135,85 @@ mv usr/local/lib/python3.11/site-packages/zuul/web/static static
 rm -Rf usr
 ```
 
+### Upgrading an image
+
+This paragraph describes the upgrade image container process, whether a major component version, a security fix or a custom patch is released.
+
+This project's containers are managed in the [containers](https://softwarefactory-project.io/r/plugins/gitiles/containers) repository.
+The first step is to clone this repository locally.
+
+The containerfiles and versions are managed with Dhall, in the `images-sf/master` directory of the repository.
+
+#### General process
+
+1. Submit your change to the `containers` repo.
+1. When this change is merged, edit the `controllers/libs/base/static/images.yaml` in the sf-operator repo to reflect
+   the new version and container id.
+1. Submit your change.
+
+#### Upgrading Zuul and Nodepool
+
+Upgrading Zuul and Nodepool requires a few extra steps:
+
+##### Containers Repo
+
+1. Ensure python dependencies are up to date by running `make update-pip-freeze`
+
+##### SF-Operator Repo
+
+*Prerequisite*: you need a local clone of [zuul](https://opendev.org/zuul/zuul) and [nodepool](https://opendev.org/zuul/nodepool).
+
+**Zuul:**
+
+1. Set your local Zuul repo to the desired tag:
+
+```sh
+cd <path/to/zuul> && git fetch --all && git checkout <tag>
+```
+
+2. Run the statsd mapper utility tool in the `sf-operator` repo:
+
+```sh
+cd <path/to/sf-operator>
+python tools/zuuldoc2statsdmapper.py -i <path/to/zuul>/doc/source/monitoring.rst controllers/static/zuul/statsd_mapping.yaml
+```
+
+3. Validate that the generated configuration can run with statsd-exporter (make sure the container version matches the one in
+   `controllers/libs/base/static/images.yaml`)
+
+```sh
+podman run --rm -v ./controllers/static/zuul/statsd_mapping.yaml:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter:v0.27.1 --statsd.mapping-config=/tmp/statsd_mapping.yaml
+```
+
+Any issue with the configuration will appear in the application logs. Otherwise, if all goes fine, the last log should be similar to
+`level=info msg="Accepting Prometheus Requests" addr=:9102`.
+
+**Nodepool:**
+
+1. Set your local Nodepool repo to the desired tag:
+
+```sh
+cd <path/to/nodepool> && git fetch --all && git checkout <tag>
+```
+
+2. Run the statsd mapper utility tool in the `sf-operator` repo:
+
+```sh
+cd <path/to/sf-operator>
+python tools/zuuldoc2statsdmapper.py -i <path/to/nodepool>/doc/source/operation.rst controllers/static/nodepool/statsd_mapping.yaml.tmpl
+```
+
+3. Validate that the generated configuration can run with statsd-exporter (make sure the container version matches the one in
+   `controllers/libs/base/static/images.yaml`)
+
+```sh
+podman run --rm -v ./controllers/static/nodepool/statsd_mapping.yaml.tmpl:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter:v0.27.1 --statsd.mapping-config=/tmp/statsd_mapping.yaml
+```
+
+Any issue with the configuration will appear in the application logs. Otherwise, if all goes fine, the last log should be similar to
+`level=info msg="Accepting Prometheus Requests" addr=:9102`.
+
+4. Uncomment the templated part at the end of `controllers/static/nodepool/statsd_mapping.yaml.tmpl`
+
+Finally, check the diffs on `controllers/static/nodepool/statsd_mapping.yaml.tmpl` and `controllers/static/zuul/statsd_mapping.yaml`
+and document any change (new or removed metrics) in the CHANGELOG.
diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md
@@ -5,7 +5,19 @@ All notable changes to this project will be documented in this file.
 ## [in development]
 
 ### Added
+
+- monitoring - Updated Zuul's statsd mappings to reflect metrics issued by the applications' current versions.
+  This adds the following metrics:
+
+  * zuul_tenant_pipeline_queue_branch
+  * zuul_executor_pct_used_inodes
+
+- development - Document the process to update a container
+
 ### Changed
+
+- security: bumped cert-manager go dependency to v1.15.4
+
 ### Deprecated
 ### Removed
 ### Fixed

diff --git a/go.mod b/go.mod
@@ -6,7 +6,7 @@ toolchain go1.22.5
 
 require (
 	github.com/apenella/go-ansible v1.3.0
-	github.com/cert-manager/cert-manager v1.15.1
+	github.com/cert-manager/cert-manager v1.15.4
 	github.com/fatih/color v1.17.0
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/google/uuid v1.6.0

diff --git a/go.sum b/go.sum
@@ -12,8 +12,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/cert-manager/cert-manager v1.15.1 h1:HSG4k2GlJ2YgTLkZfQzrArNaQpM9+ehDDg550IxAD94=
-github.com/cert-manager/cert-manager v1.15.1/go.mod h1:p98JoGv3J9JhdKU9ngsj2EhWGI6/GlU7kpjWu5lf2js=
+github.com/cert-manager/cert-manager v1.15.4 h1:FtH6BOTmkNBNRjoYSW2b80MYpUq4Zw1zbEB6flYzkiM=
+github.com/cert-manager/cert-manager v1.15.4/go.mod h1:stBge/DTvrhfQMB/93+Y62s+gQgZBsfL1o0C/4AL/mI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=

diff --git a/tools/zuuldoc2statsdmapper.py b/tools/zuuldoc2statsdmapper.py
@@ -39,6 +39,8 @@ def fix_label(label):
         return "provider"
     elif label == "diskimage_name":
         return "diskimage"
+    elif label == "diskimage name":
+        return "diskimage"
     else:
         return label
 
@@ -89,8 +91,9 @@ def stat_to_mapping(stat):
             mapping['labels'] = labels
         return [mapping, ]
 
-
-docstat_re = re.compile('^(\s*)\.\. (zuul:)?stat:: (.+)$')
+# Every metric is prefixed with "stat::" for specific rendering,
+# we use that to extract the metrics from the documentation.
+docstat_re = re.compile(r'^(.*)\.\. (zuul:)?stat:: (.+)$')
 
 
 if __name__ == "__main__":
@@ -146,16 +149,36 @@ def stat_to_mapping(stat):
     #         }
     #     )
 
-    # Drop all non-matching metrics to avoid spamming
-    mappings.append(
-        {
-         'match': '.',
-         'match_type': 'regex',
-         'action': 'drop',
-         'name': quoted('dropped')
-        }
-    )
+
+    rendered = yaml.dump({'mappings': mappings})
+
     with open(args.output_file, 'w') as o:
         o.write("# Auto-generated with zuuldoc2statsdmapper.py, please check with: \n")
         o.write("# podman run --rm -v %s:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml\n#\n" % args.output_file)
-        o.write(yaml.dump({'mappings': mappings}))
+        o.write(rendered)
+        if "nodepool" in args.input:
+            template_code = """
+# Uncomment below after testing
+# {{- range . }}
+# - name: {{ .Name }}
+#   match: {{ .Match }}
+#   help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics
+#   labels:
+# {{- range .Labels }}
+#     {{ .LabelName }}: "{{ .LabelValue -}}"
+# {{- end }}
+# {{- end }}
+# - action: drop
+#   match: .
+#   match_type: regex
+#   name: "dropped"
+"""
+            o.write(template_code)
+        # Drop all non-matching metrics to avoid spamming
+        drop = """
+- action: drop
+  match: .
+  match_type: regex
+  name: "dropped"
+"""
+        o.write(drop)