Allow setting order and hooks for test role

NOTE: this patch will require setting order for all jobs running not only
tempest

- Allow passing hook list directly to hook role
- Allow setting up stages and hooks for test operator role

Jira: https://issues.redhat.com/browse/OSPRH-10106

docs/source/usage/01_usage.md

@@ -52,8 +52,6 @@ are shared among multiple roles:
 - `cifmw_run_tests`: (Bool) Specifies whether tests should be executed.
   Defaults to false.
 - `cifmw_run_test_role`: (String) Specifies which ci-framework role will be used to run tests. Allowed options are `test_operator`, `tempest` and `shiftstack`. Defaults to `tempest`.
-- `cifmw_run_tempest`: (Bool) Specifies whether tempest tests should be run.  Notice tempest tests can be executed with either `test_operator` or `tempest` roles. Defaults to true.
-- `cifmw_run_tobiko`: (Bool) Specifies whether tobiko tests should be run. Notice tobiko tests can only be executed with the `test_operator` role. Defaults to false.
 - `cifmw_edpm_deploy_nfs`: (Bool) Specifies whether an nfs server should be deployed.
 - `cifmw_nfs_target`: (String) The Ansible inventory group where the nfs server is deployed. Defaults to `computes`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`.
 - `cifmw_nfs_network`: (String) The network the deployed nfs server will be accessible from. Defaults to `storage`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`.

playbooks/08-run-tests.yml

@@ -1,7 +1,7 @@
-- name: "Run pre_tests hooks"
-  vars:
-    step: pre_tests
-  ansible.builtin.import_playbook: ./hooks.yml
+# - name: "Run pre_tests hooks"
+#   vars:
+#     step: pre_tests
+#   ansible.builtin.import_playbook: ./hooks.yml
 
 - name: "Test playbook"
   hosts: "{{ cifmw_target_host | default('localhost') }}"
@@ -19,7 +19,7 @@
       ansible.builtin.import_role:
         name: "{{ cifmw_run_test_role | default('tempest') }}"
 
-- name: "Run post_tests hooks"
-  vars:
-    step: post_tests
-  ansible.builtin.import_playbook: ./hooks.yml
+# - name: "Run post_tests hooks"
+#   vars:
+#     step: post_tests
+#   ansible.builtin.import_playbook: ./hooks.yml

roles/run_hook/tasks/main.yml

@@ -18,7 +18,8 @@
   vars:
     _list_hooks: >-
       {{
-        hostvars[inventory_hostname][step] |
+        hostvars[inventory_hostname][step] is defined |
+        ternary(hostvars[inventory_hostname][step], hook_list) |
         default([])
       }}
     _matcher: "^{{ step }}_(.*)$"

roles/shiftstack/README.md

@@ -22,10 +22,9 @@ Role for triggering Openshift on Openstack QA automation (installation and tests
 The role is imported in the test playbook, i.e. when:
 ```
 cifmw_run_tests: true
-cifmw_run_tempest: false
 cifmw_run_test_role: shiftstack
 cifmw_run_test_shiftstack_testconfig: 4.15_ovnkubernetes_ipi_va1.yaml
 cifmw_shiftstack_qa_gerrithub_change: refs/changes/29/1188429/50 #optional
 
-$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_tempest=false cifmw_run_test_role=shiftstack cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50"
+$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50"
 ```

roles/test_operator/README.md

@@ -3,7 +3,6 @@
 Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/test-operator/).
 
 ## Parameters
-
 * `cifmw_test_operator_artifacts_basedir`: (String) Directory where we will have all test-operator related files. Default value: `{{ cifmw_basedir }}/tests/test_operator` which defaults to `~/ci-framework-data/tests/test_operator`
 * `cifmw_test_operator_namespace`: (String) Namespace inside which all the resources are created. Default value: `openstack`
 * `cifmw_test_operator_index`: (String) Full name of container image with index that contains the test-operator. Default value: `quay.io/openstack-k8s-operators/test-operator-index:latest`
@@ -23,6 +22,18 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/
 * `cifmw_test_operator_storage_class`: (String) Value for `storageClass` in generated Tempest or Tobiko CRD file. Defaults to `"lvms-local-storage"` only if `cifmw_use_lvms` is True, otherwise it defaults to `"local-storage"`.
 * `cifmw_test_operator_delete_logs_pod`: (Boolean) Delete tempest log pod created by the role at the end of the testing. Default value: `false`.
 * `cifmw_test_operator_privileged`: (Boolean) Spawn the test pods with `allowPrivilegedEscalation: true` and default linux capabilities. This is required for certain test-operator functionalities to work properly (e.g.: `extraRPMs`, certain set of tobiko tests). Default value: `true`
+* `cifmw_test_operator_stages`: (List) List of dictionaries defining the stages that should be used in the test operator role. List items options are:
+  * `name`: (String) The name of the stage. The name must be uniqe.
+  * `type`: (String) The framework name you would like to call, currently the   optinos are: tempest, ansibletest, horizontest, tobiko.
+  * `test_file`: (String) Path to the file used for testing, this file should contain the testing params for this stage. Default value: `{{ ansible_user_dir }}/ci-framework-data/artifacts/ansible-vars.yml`
+  * `pre_stage_hooks`: (List) List of pre hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format).
+  * `pre_stage_hooks`: (List) List of post hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format).
+ Default value:
+```
+cifmw_test_operator_stages:
+  - name: tempest
+    type: tempest
+```
 
 ## Tempest specific parameters
 * `cifmw_test_operator_tempest_registry`: (String) The registry where to pull tempest container. Default value: `quay.io`

roles/test_operator/defaults/main.yml

@@ -19,6 +19,9 @@
 # All variables within this role should have a prefix of "cifmw_test_operator"
 
 # Section 1: generic parameters (applied to all supported test frameworks)
+cifmw_test_operator_stages:
+  - name: tempest
+    type: tempest
 cifmw_test_operator_fail_on_test_failure: true
 cifmw_test_operator_artifacts_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/tests/test_operator"
 cifmw_test_operator_namespace: openstack

roles/test_operator/tasks/main.yml

@@ -23,13 +23,6 @@
     owner: "{{ ansible_user | default(lookup('env', 'USER')) }}"
     group: "{{ ansible_user | default(lookup('env', 'USER')) }}"
 
-- name: Set run_tempest and run_tobiko bool values
-  ansible.builtin.set_fact:
-    run_tempest: "{{ cifmw_run_tempest | default('true') | bool }}"
-    run_tobiko: "{{ cifmw_run_tobiko | default('false') | bool }}"
-    run_ansibletest: "{{ cifmw_run_ansibletest | default('false') | bool }}"
-    run_horizontest: "{{ cifmw_run_horizontest | default('false') | bool }}"
-
 - name: Ensure OperatorGroup for the test-operator is present
   kubernetes.core.k8s:
     kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
@@ -132,52 +125,12 @@
   until: csv.resources[0].status.phase | default(omit) == "Succeeded"
   when: not cifmw_test_operator_dry_run | bool
 
-- name: Run tempest job
+- name: Call test stages loop
   vars:
-    run_test_fw: tempest
-    test_operator_config: "{{ cifmw_test_operator_tempest_config }}"
-    test_operator_job_name: "{{ cifmw_test_operator_tempest_name }}"
-    test_operator_kind_name: "{{ cifmw_test_operator_tempest_kind_name }}"
-    test_operator_crd_name: "{{ cifmw_test_operator_tempest_crd_name }}"
-    test_operator_workflow: "{{ cifmw_test_operator_tempest_workflow }}"
-    test_operator_config_playbook: tempest-tests.yml
-  ansible.builtin.include_tasks: run-test-operator-job.yml
-  when: run_tempest
-
-- name: Run ansibletest job
-  vars:
-    run_test_fw: ansibletest
-    test_operator_config: "{{ cifmw_test_operator_ansibletest_config }}"
-    test_operator_job_name: "{{ cifmw_test_operator_ansibletest_name }}"
-    test_operator_kind_name: "{{ cifmw_test_operator_ansibletest_kind_name }}"
-    test_operator_crd_name: "{{ cifmw_test_operator_ansibletest_crd_name }}"
-    test_operator_workflow: "{{ cifmw_test_operator_ansibletest_workflow }}"
-  ansible.builtin.include_tasks: run-test-operator-job.yml
-  when: run_ansibletest
-
-- name: Run horizontest job
-  vars:
-    run_test_fw: horizontest
-    test_operator_config: "{{ cifmw_test_operator_horizontest_config }}"
-    test_operator_job_name: "{{ cifmw_test_operator_horizontest_name }}"
-    test_operator_kind_name: "{{ cifmw_test_operator_horizontest_kind_name }}"
-    test_operator_crd_name: "{{ cifmw_test_operator_horizontest_crd_name }}"
-    test_operator_workflow: []
-  ansible.builtin.include_tasks: run-test-operator-job.yml
-  when: run_horizontest
-
-# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end
-- name: Run tobiko job
-  vars:
-    run_test_fw: tobiko
-    test_operator_config: "{{ cifmw_test_operator_tobiko_config }}"
-    test_operator_job_name: "{{ cifmw_test_operator_tobiko_name }}"
-    test_operator_kind_name: "{{ cifmw_test_operator_tobiko_kind_name }}"
-    test_operator_crd_name: "{{ cifmw_test_operator_tobiko_crd_name }}"
-    test_operator_workflow: "{{ cifmw_test_operator_tobiko_workflow }}"
-    test_operator_config_playbook: tobiko-tests.yml
-  ansible.builtin.include_tasks: run-test-operator-job.yml
-  when: run_tobiko
+    _stage_vars: "{{ item }}"
+  ansible.builtin.include_tasks: stages.yml
+  when: not cifmw_test_operator_dry_run | bool
+  loop: "{{ cifmw_test_operator_stages }}"
 
 - name: Delete all resources created by the role
   ansible.builtin.include_tasks: cleanup.yml

roles/test_operator/tasks/runners/ansibletest_runner.yml

@@ -0,0 +1,9 @@
+- name: Run ansibletest job
+  vars:
+    run_test_fw: ansibletest
+    test_operator_config: "{{ _stage_var_file.cifmw_test_operator_ansibletest_config | default(cifmw_test_operator_ansibletest_config) }}"
+    test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_name | default(cifmw_test_operator_ansibletest_name) }}-{{ _stage_vars.name }}"
+    test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_kind_name | default(cifmw_test_operator_ansibletest_kind_name) }}"
+    test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_ansibletest_crd_name | default(cifmw_test_operator_ansibletest_crd_name) }}"
+    test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_ansibletest_workflow | default(cifmw_test_operator_ansibletest_workflow) }}"
+  ansible.builtin.include_tasks: run-test-operator-job.yml

roles/test_operator/tasks/runners/horizontest_runner.yml

@@ -0,0 +1,9 @@
+- name: Run horizontest job
+  vars:
+    run_test_fw: horizontest
+    test_operator_config: "{{ _stage_var_file.cifmw_test_operator_horizontest_config | default(cifmw_test_operator_horizontest_config) }}"
+    test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_name | default(cifmw_test_operator_horizontest_name) }}"
+    test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_kind_name | default(cifmw_test_operator_horizontest_kind_name) }}"
+    test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_horizontest_crd_name | default(cifmw_test_operator_horizontest_crd_name) }}"
+    test_operator_workflow: []
+  ansible.builtin.include_tasks: run-test-operator-job.yml

roles/test_operator/tasks/runners/tempest_runner.yml

@@ -0,0 +1,11 @@
+
+- name: Run tempest job
+  vars:
+    run_test_fw: tempest
+    test_operator_config: "{{ _stage_var_file.cifmw_test_operator_tempest_config | default(cifmw_test_operator_tempest_config) }}"
+    test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_tempest_name | default(cifmw_test_operator_tempest_name) }}-{{ _stage_vars.name }}"
+    test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_tempest_kind_name | default(cifmw_test_operator_tempest_kind_name) }}"
+    test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_tempest_crd_name | default(cifmw_test_operator_tempest_crd_name) }}"
+    test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_tempest_workflow | default(cifmw_test_operator_tempest_workflow) }}"
+    test_operator_config_playbook: tempest-tests.yml
+  ansible.builtin.include_tasks: run-test-operator-job.yml

roles/test_operator/tasks/runners/tobiko_runner.yml

@@ -0,0 +1,11 @@
+# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end
+- name: Run tobiko job
+  vars:
+    run_test_fw: tobiko
+    test_operator_config: "{{ _stage_var_file.cifmw_test_operator_tobiko_config | default(cifmw_test_operator_tobiko_config) }}"
+    test_operator_job_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_name |  default(cifmw_test_operator_tobiko_name) }}-{{ _stage_vars.name }}"
+    test_operator_kind_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_kind_name | default(cifmw_test_operator_tobiko_kind_name) }}"
+    test_operator_crd_name: "{{ _stage_var_file.cifmw_test_operator_tobiko_crd_name |  default(cifmw_test_operator_tobiko_crd_name) }}"
+    test_operator_workflow: "{{ _stage_var_file.cifmw_test_operator_tobiko_workflow |  default(cifmw_test_operator_tobiko_workflow) }}"
+    test_operator_config_playbook: tobiko-tests.yml
+  ansible.builtin.include_tasks: run-test-operator-job.yml

roles/test_operator/tasks/stages.yml

@@ -0,0 +1,30 @@
+
+- debug:
+    msg: "Run stage {{ _stage_vars.name }}"
+
+- name: "Call pre stage hooks for test operator stage {{ _stage_vars.name }}"
+  vars:
+    step: none
+    hook_list: "{{ _stage_vars.pre_stage_hooks }}"
+  ansible.builtin.import_role:
+    name: run_hook
+  when:
+  - not cifmw_test_operator_dry_run | bool
+  - _stage_vars.pre_stage_hooks is defined
+
+- name: Include stage var file
+  ansible.builtin.include_vars:
+    file: "{{ _stage_vars.test_file | default(ansible_user_dir ~ '/ci-framework-data/artifacts/ansible-vars.yml') }}"
+    name: _stage_var_file
+
+- name: "Call {{ _stage_vars.type }} runner"
+  ansible.builtin.include_tasks: "runners/{{ _stage_vars.type }}_runner.yml"
+
+- name: "Call post stage hooks for test operator stage {{ _stage_vars.name }}"
+  vars:
+    step: _stage_vars.post_stage_hooks
+  ansible.builtin.import_role:
+    name: run_hook
+  when:
+  - not cifmw_test_operator_dry_run | bool
+  - _stage_vars.post_stage_hooks is defined

scenarios/reproducers/bgp-4-racks-3-ocps.yml

@@ -1,6 +1,5 @@
 ---
 cifmw_run_tests: true
-cifmw_run_tempest: true
 cifmw_run_test_role: test_operator
 cifmw_test_operator_timeout: 7200
 cifmw_test_operator_tempest_include_list: |

scenarios/reproducers/va-common.yml

@@ -29,7 +29,6 @@ cifmw_libvirt_manager_pub_net: ocpbm
 
 # Run tests
 cifmw_run_tests: true
-cifmw_run_tempest: true
 cifmw_run_test_role: test_operator
 cifmw_test_operator_timeout: 7200
 cifmw_test_operator_tempest_include_list: |

zuul.d/architecture-jobs.yaml

@@ -53,7 +53,6 @@
       cifmw_openshift_setup_skip_internal_registry_tls_verify: true
       cifmw_networking_env_def_file: "networking-env-definition.yml"
       cifmw_networking_mapper_networking_env_def_path: "/etc/ci/env/networking-environment-definition.yml"
-      cifmw_run_tempest: true
       cifmw_run_test_role: test_operator
       cifmw_run_tests: true
       cifmw_test_operator_concurrency: 4