diff --git a/controllers/static/nodepool/statsd_mapping.yaml.tmpl b/controllers/static/nodepool/statsd_mapping.yaml.tmpl index 8e7365c..35b0e4b 100644 --- a/controllers/static/nodepool/statsd_mapping.yaml.tmpl +++ b/controllers/static/nodepool/statsd_mapping.yaml.tmpl @@ -1,5 +1,5 @@ # Auto-generated with zuuldoc2statsdmapper.py, please check with: -# podman run --rm -v controllers/static/nodepool/statsd_mapping.yaml:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml +# podman run --rm -v controllers/static/nodepool/statsd_mapping.yaml.tmpl:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml # mappings: - help: Description at https://zuul-ci.org/docs/nodepool/latest/operation.html#stat-nodepool.dib_image_build...size @@ -200,15 +200,22 @@ mappings: hostname: "$1" match: nodepool.launcher.*.zk.image_cache.playback_queue name: nodepool_launcher_zk_image_cache_playback_queue -{{- range . }} -- name: {{ .Name }} - match: {{ .Match }} - help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics - labels: -{{- range .Labels }} - {{ .LabelName }}: "{{ .LabelValue -}}" -{{- end }} -{{- end }} + +# Uncomment below after testing +# {{- range . }} +# - name: {{ .Name }} +# match: {{ .Match }} +# help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics +# labels: +# {{- range .Labels }} +# {{ .LabelName }}: "{{ .LabelValue -}}" +# {{- end }} +# {{- end }} +# - action: drop +# match: . +# match_type: regex +# name: "dropped" + - action: drop match: . match_type: regex diff --git a/controllers/static/zuul/statsd_mapping.yaml b/controllers/static/zuul/statsd_mapping.yaml index e45382b..b87400c 100644 --- a/controllers/static/zuul/statsd_mapping.yaml +++ b/controllers/static/zuul/statsd_mapping.yaml @@ -179,46 +179,56 @@ mappings: tenant: "$1" match: zuul.tenant.*.pipeline.*.queue name: zuul_tenant_pipeline_queue -- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue. +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue. labels: pipeline: "$2" queue: "$3" tenant: "$1" match: zuul.tenant.*.pipeline.*.queue.* name: zuul_tenant_pipeline_queue -- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..current_changes +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..current_changes labels: pipeline: "$2" queue: "$3" tenant: "$1" match: zuul.tenant.*.pipeline.*.queue.*.current_changes name: zuul_tenant_pipeline_queue_current_changes -- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..window +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..window labels: pipeline: "$2" queue: "$3" tenant: "$1" match: zuul.tenant.*.pipeline.*.queue.*.window name: zuul_tenant_pipeline_queue_window -- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..resident_time +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..resident_time labels: pipeline: "$2" queue: "$3" tenant: "$1" match: zuul.tenant.*.pipeline.*.queue.*.resident_time name: zuul_tenant_pipeline_queue_resident_time -- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..total_changes +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..total_changes labels: pipeline: "$2" queue: "$3" tenant: "$1" match: zuul.tenant.*.pipeline.*.queue.*.total_changes name: zuul_tenant_pipeline_queue_total_changes +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..branch + labels: + pipeline: "$2" + queue: "$3" + tenant: "$1" + match: zuul.tenant.*.pipeline.*.queue.*.branch + name: zuul_tenant_pipeline_queue_branch +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..queue..branch. + labels: + branch: "$4" + pipeline: "$2" + queue: "$3" + tenant: "$1" + match: zuul.tenant.*.pipeline.*.queue.*.branch.* + name: zuul_tenant_pipeline_queue_branch - help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.tenant..pipeline..project labels: pipeline: "$2" @@ -464,6 +474,11 @@ mappings: executor: "$1" match: zuul.executor.*.pct_used_hdd name: zuul_executor_pct_used_hdd +- help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.executor..pct_used_inodes + labels: + executor: "$1" + match: zuul.executor.*.pct_used_inodes + name: zuul_executor_pct_used_inodes - help: Description at https://zuul-ci.org/docs/zuul/latest/monitoring.html#stat-zuul.executor..pct_used_ram labels: executor: "$1" @@ -659,6 +674,7 @@ mappings: hostname: "$1" match: zuul.web.server.*.streamers name: zuul_web_server_streamers + - action: drop match: . match_type: regex diff --git a/doc/developer/images.md b/doc/developer/images.md index 58d284d..2366a4f 100644 --- a/doc/developer/images.md +++ b/doc/developer/images.md @@ -11,6 +11,7 @@ for development purposes. 1. [Modify an existing image](#modify-an-existing-image) 1. [Create and use an image from a Containerfile](#create-and-use-an-image-from-a-containerfile) 1. [Edit Zuul source code and mount in a pod](#edit-the-zuul-source-code-and-mount-in-a-pod) +1. [Upgrading an image](#upgrading-an-image) ## Root access inside containers @@ -134,3 +135,85 @@ mv usr/local/lib/python3.11/site-packages/zuul/web/static static rm -Rf usr ``` +### Upgrading an image + +This paragraph describes the upgrade image container process, whether a major component version, a security fix or a custom patch is released. + +This project's containers are managed in the [containers](https://softwarefactory-project.io/r/plugins/gitiles/containers) repository. +The first step is to clone this repository locally. + +The containerfiles and versions are managed with Dhall, in the `images-sf/master` directory of the repository. + +#### General process + +1. Submit your change to the `containers` repo. +1. When this change is merged, edit the `controllers/libs/base/static/images.yaml` in the sf-operator repo to reflect + the new version and container id. +1. Submit your change. + +#### Upgrading Zuul and Nodepool + +Upgrading Zuul and Nodepool requires a few extra steps: + +##### Containers Repo + +1. Ensure python dependencies are up to date by running `make update-pip-freeze` + +##### SF-Operator Repo + +*Prerequisite*: you need a local clone of [zuul](https://opendev.org/zuul/zuul) and [nodepool](https://opendev.org/zuul/nodepool). + +**Zuul:** + +1. Set your local Zuul repo to the desired tag: + +```sh +cd && git fetch --all && git checkout +``` + +2. Run the statsd mapper utility tool in the `sf-operator` repo: + +```sh +cd +python tools/zuuldoc2statsdmapper.py -i /doc/source/monitoring.rst controllers/static/zuul/statsd_mapping.yaml +``` + +3. Validate that the generated configuration can run with statsd-exporter (make sure the container version matches the one in + `controllers/libs/base/static/images.yaml`) + +```sh +podman run --rm -v ./controllers/static/zuul/statsd_mapping.yaml:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter:v0.27.1 --statsd.mapping-config=/tmp/statsd_mapping.yaml +``` + +Any issue with the configuration will appear in the application logs. Otherwise, if all goes fine, the last log should be similar to +`level=info msg="Accepting Prometheus Requests" addr=:9102`. + +**Nodepool:** + +1. Set your local Nodepool repo to the desired tag: + +```sh +cd && git fetch --all && git checkout +``` + +2. Run the statsd mapper utility tool in the `sf-operator` repo: + +```sh +cd +python tools/zuuldoc2statsdmapper.py -i /doc/source/operation.rst controllers/static/nodepool/statsd_mapping.yaml.tmpl +``` + +3. Validate that the generated configuration can run with statsd-exporter (make sure the container version matches the one in + `controllers/libs/base/static/images.yaml`) + +```sh +podman run --rm -v ./controllers/static/nodepool/statsd_mapping.yaml.tmpl:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter:v0.27.1 --statsd.mapping-config=/tmp/statsd_mapping.yaml +``` + +Any issue with the configuration will appear in the application logs. Otherwise, if all goes fine, the last log should be similar to +`level=info msg="Accepting Prometheus Requests" addr=:9102`. + +4. Uncomment the templated part at the end of `controllers/static/nodepool/statsd_mapping.yaml.tmpl` + +Finally, check the diffs on `controllers/static/nodepool/statsd_mapping.yaml.tmpl` and `controllers/static/zuul/statsd_mapping.yaml` +and document any change (new or removed metrics) in the CHANGELOG. \ No newline at end of file diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md index 6d72b6a..8ff3607 100644 --- a/doc/reference/CHANGELOG.md +++ b/doc/reference/CHANGELOG.md @@ -5,7 +5,19 @@ All notable changes to this project will be documented in this file. ## [in development] ### Added + +- monitoring - Updated Zuul's statsd mappings to reflect metrics issued by the applications' current versions. + This adds the following metrics: + + * zuul_tenant_pipeline_queue_branch + * zuul_executor_pct_used_inodes + +- development - Document the process to update a container + ### Changed + +- security: bumped cert-manager go dependency to v1.15.4 + ### Deprecated ### Removed ### Fixed diff --git a/go.mod b/go.mod index c1cea0c..1580e34 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.5 require ( github.com/apenella/go-ansible v1.3.0 - github.com/cert-manager/cert-manager v1.15.1 + github.com/cert-manager/cert-manager v1.15.4 github.com/fatih/color v1.17.0 github.com/go-sql-driver/mysql v1.8.1 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index b82eeea..f523cf7 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cert-manager/cert-manager v1.15.1 h1:HSG4k2GlJ2YgTLkZfQzrArNaQpM9+ehDDg550IxAD94= -github.com/cert-manager/cert-manager v1.15.1/go.mod h1:p98JoGv3J9JhdKU9ngsj2EhWGI6/GlU7kpjWu5lf2js= +github.com/cert-manager/cert-manager v1.15.4 h1:FtH6BOTmkNBNRjoYSW2b80MYpUq4Zw1zbEB6flYzkiM= +github.com/cert-manager/cert-manager v1.15.4/go.mod h1:stBge/DTvrhfQMB/93+Y62s+gQgZBsfL1o0C/4AL/mI= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/tools/zuuldoc2statsdmapper.py b/tools/zuuldoc2statsdmapper.py index fe1e380..2a8cf87 100644 --- a/tools/zuuldoc2statsdmapper.py +++ b/tools/zuuldoc2statsdmapper.py @@ -39,6 +39,8 @@ def fix_label(label): return "provider" elif label == "diskimage_name": return "diskimage" + elif label == "diskimage name": + return "diskimage" else: return label @@ -89,8 +91,9 @@ def stat_to_mapping(stat): mapping['labels'] = labels return [mapping, ] - -docstat_re = re.compile('^(\s*)\.\. (zuul:)?stat:: (.+)$') +# Every metric is prefixed with "stat::" for specific rendering, +# we use that to extract the metrics from the documentation. +docstat_re = re.compile(r'^(.*)\.\. (zuul:)?stat:: (.+)$') if __name__ == "__main__": @@ -146,16 +149,36 @@ def stat_to_mapping(stat): # } # ) - # Drop all non-matching metrics to avoid spamming - mappings.append( - { - 'match': '.', - 'match_type': 'regex', - 'action': 'drop', - 'name': quoted('dropped') - } - ) + + rendered = yaml.dump({'mappings': mappings}) + with open(args.output_file, 'w') as o: o.write("# Auto-generated with zuuldoc2statsdmapper.py, please check with: \n") o.write("# podman run --rm -v %s:/tmp/statsd_mapping.yaml:z docker.io/prom/statsd-exporter --statsd.mapping-config=/tmp/statsd_mapping.yaml\n#\n" % args.output_file) - o.write(yaml.dump({'mappings': mappings})) + o.write(rendered) + if "nodepool" in args.input: + template_code = """ +# Uncomment below after testing +# {{- range . }} +# - name: {{ .Name }} +# match: {{ .Match }} +# help: {{ .Help }}, see https://zuul-ci.org/docs/nodepool/latest/operation.html#openstack-api-metrics +# labels: +# {{- range .Labels }} +# {{ .LabelName }}: "{{ .LabelValue -}}" +# {{- end }} +# {{- end }} +# - action: drop +# match: . +# match_type: regex +# name: "dropped" +""" + o.write(template_code) + # Drop all non-matching metrics to avoid spamming + drop = """ +- action: drop + match: . + match_type: regex + name: "dropped" +""" + o.write(drop)