[ci] Add a pre_deploy hook to create the oscp kustomization to enable cloudkitty

* add s3storageConfig secret
* disable autoscaling and logging
* Enable metricsStorage in telemetry to provide prometheus for storage
* Configure storageClass for CloudKitty in configure-cloudkitty hook
  Set the storageClass to crc-csi-hostpath-provisioner for CloudKitty
  deployment in CRC-based CI jobs. This prevents the need to increase
  PVC allocations in install_yamls by using the default storage class
  available in CRC, which allocates storage as required.

Author: elfiesmelfie

ci/configure-cloudkitty.yml

@@ -0,0 +1,44 @@
+---
+- name: "Create the kustomization for deploying CloudKitty"
+  hosts: "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: false
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  tasks:
+    - name: Copy controlplane kustomization
+      ansible.builtin.copy:
+        dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-controlplane-cloudkitty.yaml"
+        content: |-
+          apiVersion: kustomize.config.k8s.io/v1beta1
+          kind: Kustomization
+          namespace: openstack
+          patches:
+          - patch: |-
+              apiVersion: core.openstack.org/v1beta1
+              kind: OpenStackControlPlane
+              metadata:
+                name: unused
+              spec:
+                # Set overall storage class so we don't need to increase the
+                # number of PVCs that install_yamls creates
+                # this is only applicable to crc-based jobs, it is not in
+                # openshift by default, but is included in the crc distribution
+                storageClass: crc-csi-hostpath-provisioner
+                telemetry:
+                  enabled: true
+                  template:
+                    logging:
+                      enabled: false
+                    autoscaling:
+                      enabled: false
+                    cloudkitty:
+                      enabled: true
+                      s3StorageConfig:
+                        secret:
+                          type: "s3"
+                          name: "cloudkitty-loki-s3"
+                    metricStorage:
+                      enabled: true
+            target:
+              kind: OpenStackControlPlane

ci/vars-cloudkitty-tempest.yml

@@ -1,6 +1,26 @@
 ---
+cifmw_deploy_obs: true
+cifmw_openshift_obs_definition:
+  apiVersion: operators.coreos.com/v1alpha1
+  kind: Subscription
+  metadata:
+    name: observability-operator
+    namespace: openshift-operators
+  spec:
+    channel: stable
+    installPlanApproval: Automatic
+    name: cluster-observability-operator
+    source: redhat-operators
+    sourceNamespace: openshift-marketplace
+
+pre_deploy_kustomize_cloudkitty:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml"
+  type: playbook
+
+# test cloudkitty
 cifmw_run_tests: true
 cifmw_run_test_role: test_operator
+# TODO: Consider switching to podified-master-centos10 for features that patch master
 cifmw_test_operator_tempest_namespace: podified-antelope-centos9
 # cloudkitty tempest plugin is not part of the tempest rpm.
 # https://review.rdoproject.org/cgit/openstack/tempest-distgit/tree/openstack-tempest.spec
@@ -11,6 +31,7 @@ cifmw_test_operator_tempest_container: openstack-tempest-all
 cifmw_test_operator_tempest_image_tag: 'current-podified'
 # This value is used to populate the `tempestconfRun` parameter of the Tempest CR: https://openstack-k8s-operators.github.io/test-operator/crds.html#tempest-custom-resource
 # https://github.com/openstack-k8s-operators/ci-framework/blob/main/roles/test_operator/defaults/main.yml
+# TODO: Refine this tempest config
 tempest_conf:
   overrides: |
       validation.run_validation true
@@ -29,10 +50,9 @@ cifmw_test_operator_tempest_include_list: |
   ^tempest.*\[.*\bsmoke\b.*\]
   cloudkitty_tempest_plugin.*
   telemetry_tempest_plugin.*
-cifmw_test_operator_tempest_exclude_list: |
-  telemetry_tempest_plugin.scenario.test_telemetry_integration_prometheus.PrometheusGabbiTest.test_autoscaling
-
+# TODO: update this to allow multiple external plugins to be listed with Depends-On.
+# Potentially, this can be done via the meta content provider, by adding the tempest images to the list.
 external_plugin: "opendev.org/openstack/cloudkitty-tempest-plugin"
 change_item: "{{ zuul['items'] | selectattr('project.canonical_name', 'equalto', external_plugin) }}"
-# This is a workaround because CK tempest is not packages in RDO. Typically, the default would be [], since we would not require an external installation.
+# WORKAROUND: CloudKitty tempest is not packaged in RDO. Typically, the default would be [], since we would not require an external installation.
 cifmw_test_operator_tempest_external_plugin: "{{ [ {'repository': 'https://' + external_plugin + '.git'} ] if change_item | length < 1 else [ { 'repository': 'https://' + external_plugin + '.git', 'changeRepository': 'https://review' + external_plugin, 'changeRefspec': [ 'refs/changes', change_item[0].change[-2:], change_item[0].change, change_item[0].patchset ] | join('/') } ] }}"