diff --git a/docs/source/usage/01_usage.md b/docs/source/usage/01_usage.md index 70707e7f17..1c0698a024 100644 --- a/docs/source/usage/01_usage.md +++ b/docs/source/usage/01_usage.md @@ -52,8 +52,6 @@ are shared among multiple roles: - `cifmw_run_tests`: (Bool) Specifies whether tests should be executed. Defaults to false. - `cifmw_run_test_role`: (String) Specifies which ci-framework role will be used to run tests. Allowed options are `test_operator`, `tempest` and `shiftstack`. Defaults to `tempest`. -- `cifmw_run_tempest`: (Bool) Specifies whether tempest tests should be run. Notice tempest tests can be executed with either `test_operator` or `tempest` roles. Defaults to true. -- `cifmw_run_tobiko`: (Bool) Specifies whether tobiko tests should be run. Notice tobiko tests can only be executed with the `test_operator` role. Defaults to false. - `cifmw_edpm_deploy_nfs`: (Bool) Specifies whether an nfs server should be deployed. - `cifmw_nfs_target`: (String) The Ansible inventory group where the nfs server is deployed. Defaults to `computes`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`. - `cifmw_nfs_network`: (String) The network the deployed nfs server will be accessible from. Defaults to `storage`. Only has an effect if `cifmw_edpm_deploy_nfs` is set to `true`. diff --git a/playbooks/08-run-tests.yml b/playbooks/08-run-tests.yml index 31bf5ee818..ea821d1224 100644 --- a/playbooks/08-run-tests.yml +++ b/playbooks/08-run-tests.yml @@ -1,7 +1,7 @@ -- name: "Run pre_tests hooks" - vars: - step: pre_tests - ansible.builtin.import_playbook: ./hooks.yml +# - name: "Run pre_tests hooks" +# vars: +# step: pre_tests +# ansible.builtin.import_playbook: ./hooks.yml - name: "Test playbook" hosts: "{{ cifmw_target_host | default('localhost') }}" @@ -19,7 +19,7 @@ ansible.builtin.import_role: name: "{{ cifmw_run_test_role | default('tempest') }}" -- name: "Run post_tests hooks" - vars: - step: post_tests - ansible.builtin.import_playbook: ./hooks.yml +# - name: "Run post_tests hooks" +# vars: +# step: post_tests +# ansible.builtin.import_playbook: ./hooks.yml diff --git a/roles/run_hook/tasks/main.yml b/roles/run_hook/tasks/main.yml index bd39f87e53..00cdd3c1cd 100644 --- a/roles/run_hook/tasks/main.yml +++ b/roles/run_hook/tasks/main.yml @@ -18,7 +18,8 @@ vars: _list_hooks: >- {{ - hostvars[inventory_hostname][step] | + hostvars[inventory_hostname][step] is defined | + ternary(hostvars[inventory_hostname][step], hook_list) | default([]) }} _matcher: "^{{ step }}_(.*)$" diff --git a/roles/test_operator/README.md b/roles/test_operator/README.md index 83adf6cf09..25e36c93ac 100644 --- a/roles/test_operator/README.md +++ b/roles/test_operator/README.md @@ -3,7 +3,6 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/test-operator/). ## Parameters - * `cifmw_test_operator_artifacts_basedir`: (String) Directory where we will have all test-operator related files. Default value: `{{ cifmw_basedir }}/tests/test_operator` which defaults to `~/ci-framework-data/tests/test_operator` * `cifmw_test_operator_namespace`: (String) Namespace inside which all the resources are created. Default value: `openstack` * `cifmw_test_operator_index`: (String) Full name of container image with index that contains the test-operator. Default value: `quay.io/openstack-k8s-operators/test-operator-index:latest` @@ -23,6 +22,18 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/ * `cifmw_test_operator_storage_class`: (String) Value for `storageClass` in generated Tempest or Tobiko CRD file. Defaults to `"lvms-local-storage"` only if `cifmw_use_lvms` is True, otherwise it defaults to `"local-storage"`. * `cifmw_test_operator_delete_logs_pod`: (Boolean) Delete tempest log pod created by the role at the end of the testing. Default value: `false`. * `cifmw_test_operator_privileged`: (Boolean) Spawn the test pods with `allowPrivilegedEscalation: true` and default linux capabilities. This is required for certain test-operator functionalities to work properly (e.g.: `extraRPMs`, certain set of tobiko tests). Default value: `true` +* `cifmw_test_operator_stages`: (List) List of dictionaries defining the stages that should be used in the test operator role. List items options are: + * `name`: (String) The name of the stage. + * `type`: (String) The framework name you would like to call, currently the optinos are: tempest, ansibletest, horizontest, tobiko. + * `test_file`: (String) Path to the file used for testing, this file should contain the testing params for this stage. Default value: `{{ ansible_user_dir }}/ci-framework-data/artifacts/ansible-vars.yml` + * `pre_stage_hooks`: (List) List of pre hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). + * `pre_stage_hooks`: (List) List of post hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). + Default value: +``` +cifmw_test_operator_stages: + - name: tempest + type: tempest +``` ## Tempest specific parameters * `cifmw_test_operator_tempest_registry`: (String) The registry where to pull tempest container. Default value: `quay.io` diff --git a/roles/test_operator/defaults/main.yml b/roles/test_operator/defaults/main.yml index da88f8ab44..ec00baa4cd 100644 --- a/roles/test_operator/defaults/main.yml +++ b/roles/test_operator/defaults/main.yml @@ -19,6 +19,9 @@ # All variables within this role should have a prefix of "cifmw_test_operator" # Section 1: generic parameters (applied to all supported test frameworks) +cifmw_test_operator_stages: + - name: tempest + type: tempest cifmw_test_operator_fail_on_test_failure: true cifmw_test_operator_artifacts_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/tests/test_operator" cifmw_test_operator_namespace: openstack diff --git a/roles/test_operator/tasks/main.yml b/roles/test_operator/tasks/main.yml index b745d7da2c..2852f6ea74 100644 --- a/roles/test_operator/tasks/main.yml +++ b/roles/test_operator/tasks/main.yml @@ -23,13 +23,6 @@ owner: "{{ ansible_user | default(lookup('env', 'USER')) }}" group: "{{ ansible_user | default(lookup('env', 'USER')) }}" -- name: Set run_tempest and run_tobiko bool values - ansible.builtin.set_fact: - run_tempest: "{{ cifmw_run_tempest | default('true') | bool }}" - run_tobiko: "{{ cifmw_run_tobiko | default('false') | bool }}" - run_ansibletest: "{{ cifmw_run_ansibletest | default('false') | bool }}" - run_horizontest: "{{ cifmw_run_horizontest | default('false') | bool }}" - - name: Ensure OperatorGroup for the test-operator is present kubernetes.core.k8s: kubeconfig: "{{ cifmw_openshift_kubeconfig }}" @@ -132,52 +125,12 @@ until: csv.resources[0].status.phase | default(omit) == "Succeeded" when: not cifmw_test_operator_dry_run | bool -- name: Run tempest job +- name: Call test stages loop vars: - run_test_fw: tempest - test_operator_config: "{{ cifmw_test_operator_tempest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_tempest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_tempest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_tempest_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_tempest_workflow }}" - test_operator_config_playbook: tempest-tests.yml - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_tempest - -- name: Run ansibletest job - vars: - run_test_fw: ansibletest - test_operator_config: "{{ cifmw_test_operator_ansibletest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_ansibletest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_ansibletest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_ansibletest_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_ansibletest_workflow }}" - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_ansibletest - -- name: Run horizontest job - vars: - run_test_fw: horizontest - test_operator_config: "{{ cifmw_test_operator_horizontest_config }}" - test_operator_job_name: "{{ cifmw_test_operator_horizontest_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_horizontest_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_horizontest_crd_name }}" - test_operator_workflow: [] - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_horizontest - -# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end -- name: Run tobiko job - vars: - run_test_fw: tobiko - test_operator_config: "{{ cifmw_test_operator_tobiko_config }}" - test_operator_job_name: "{{ cifmw_test_operator_tobiko_name }}" - test_operator_kind_name: "{{ cifmw_test_operator_tobiko_kind_name }}" - test_operator_crd_name: "{{ cifmw_test_operator_tobiko_crd_name }}" - test_operator_workflow: "{{ cifmw_test_operator_tobiko_workflow }}" - test_operator_config_playbook: tobiko-tests.yml - ansible.builtin.include_tasks: run-test-operator-job.yml - when: run_tobiko + stage_vars: "{{ item }}" + ansible.builtin.include_tasks: stages.yml + when: not cifmw_test_operator_dry_run | bool + loop: "{{ cifmw_test_operator_stages }}" - name: Delete all resources created by the role ansible.builtin.include_tasks: cleanup.yml diff --git a/roles/test_operator/tasks/runners/ansibletest_runner.yml b/roles/test_operator/tasks/runners/ansibletest_runner.yml new file mode 100644 index 0000000000..719385b834 --- /dev/null +++ b/roles/test_operator/tasks/runners/ansibletest_runner.yml @@ -0,0 +1,9 @@ +- name: Run ansibletest job + vars: + run_test_fw: ansibletest + test_operator_config: "{{ cifmw_test_operator_ansibletest_config }}" + test_operator_job_name: "{{ cifmw_test_operator_ansibletest_name }}" + test_operator_kind_name: "{{ cifmw_test_operator_ansibletest_kind_name }}" + test_operator_crd_name: "{{ cifmw_test_operator_ansibletest_crd_name }}" + test_operator_workflow: "{{ cifmw_test_operator_ansibletest_workflow }}" + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/horizontest_runner.yml b/roles/test_operator/tasks/runners/horizontest_runner.yml new file mode 100644 index 0000000000..76f6234e93 --- /dev/null +++ b/roles/test_operator/tasks/runners/horizontest_runner.yml @@ -0,0 +1,9 @@ +- name: Run horizontest job + vars: + run_test_fw: horizontest + test_operator_config: "{{ cifmw_test_operator_horizontest_config }}" + test_operator_job_name: "{{ cifmw_test_operator_horizontest_name }}" + test_operator_kind_name: "{{ cifmw_test_operator_horizontest_kind_name }}" + test_operator_crd_name: "{{ cifmw_test_operator_horizontest_crd_name }}" + test_operator_workflow: [] + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/tempest_runner.yml b/roles/test_operator/tasks/runners/tempest_runner.yml new file mode 100644 index 0000000000..a819c7c9e9 --- /dev/null +++ b/roles/test_operator/tasks/runners/tempest_runner.yml @@ -0,0 +1,10 @@ +- name: Run tempest job + vars: + run_test_fw: tempest + test_operator_config: "{{ cifmw_test_operator_tempest_config }}" + test_operator_job_name: "{{ cifmw_test_operator_tempest_name }}" + test_operator_kind_name: "{{ cifmw_test_operator_tempest_kind_name }}" + test_operator_crd_name: "{{ cifmw_test_operator_tempest_crd_name }}" + test_operator_workflow: "{{ cifmw_test_operator_tempest_workflow }}" + test_operator_config_playbook: tempest-tests.yml + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/runners/tobiko_runner.yml b/roles/test_operator/tasks/runners/tobiko_runner.yml new file mode 100644 index 0000000000..185c6a79f2 --- /dev/null +++ b/roles/test_operator/tasks/runners/tobiko_runner.yml @@ -0,0 +1,11 @@ +# Since Tobiko may include disruptive tests (faults), it is better to execute it at the end +- name: Run tobiko job + vars: + run_test_fw: tobiko + test_operator_config: "{{ cifmw_test_operator_tobiko_config }}" + test_operator_job_name: "{{ cifmw_test_operator_tobiko_name }}" + test_operator_kind_name: "{{ cifmw_test_operator_tobiko_kind_name }}" + test_operator_crd_name: "{{ cifmw_test_operator_tobiko_crd_name }}" + test_operator_workflow: "{{ cifmw_test_operator_tobiko_workflow }}" + test_operator_config_playbook: tobiko-tests.yml + ansible.builtin.include_tasks: run-test-operator-job.yml diff --git a/roles/test_operator/tasks/stages.yml b/roles/test_operator/tasks/stages.yml new file mode 100644 index 0000000000..0661f978ef --- /dev/null +++ b/roles/test_operator/tasks/stages.yml @@ -0,0 +1,29 @@ + +- debug: + msg: "Run stage {{ stage_vars.name }}" + +- name: "Call pre stage hooks for test operator stage {{ stage_vars.name }}" + vars: + step: none + hook_list: "{{ stage_vars.pre_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook + when: + - not cifmw_test_operator_dry_run | bool + - stage_vars.pre_stage_hooks is defined + +- name: Include stage var file + ansible.builtin.include_vars: + file: "{{ stage_vars.test_file | default(ansible_user_dir ~ '/ci-framework-data/artifacts/ansible-vars.yml') }}" + +- name: "Call {{ stage_vars.type }} runner" + ansible.builtin.include_tasks: "runners/{{ stage_vars.type }}_runner.yml" + +- name: "Call post stage hooks for test operator stage {{ stage_vars.name }}" + vars: + step: stage_vars.post_stage_hooks + ansible.builtin.import_role: + name: run_hook + when: + - not cifmw_test_operator_dry_run | bool + - stage_vars.post_stage_hooks is defined diff --git a/scenarios/reproducers/bgp-4-racks-3-ocps.yml b/scenarios/reproducers/bgp-4-racks-3-ocps.yml index 7cb1204ca5..3b73dd4d93 100644 --- a/scenarios/reproducers/bgp-4-racks-3-ocps.yml +++ b/scenarios/reproducers/bgp-4-racks-3-ocps.yml @@ -1,6 +1,5 @@ --- cifmw_run_tests: true -cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_test_operator_timeout: 7200 cifmw_test_operator_tempest_include_list: | diff --git a/scenarios/reproducers/va-common.yml b/scenarios/reproducers/va-common.yml index b23cd96a4a..5b92bcb41c 100644 --- a/scenarios/reproducers/va-common.yml +++ b/scenarios/reproducers/va-common.yml @@ -29,7 +29,6 @@ cifmw_libvirt_manager_pub_net: ocpbm # Run tests cifmw_run_tests: true -cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_test_operator_timeout: 7200 cifmw_test_operator_tempest_include_list: | diff --git a/zuul.d/architecture-jobs.yaml b/zuul.d/architecture-jobs.yaml index 4edcdaf601..08bbcc5c8d 100644 --- a/zuul.d/architecture-jobs.yaml +++ b/zuul.d/architecture-jobs.yaml @@ -53,7 +53,6 @@ cifmw_openshift_setup_skip_internal_registry_tls_verify: true cifmw_networking_env_def_file: "networking-env-definition.yml" cifmw_networking_mapper_networking_env_def_path: "/etc/ci/env/networking-environment-definition.yml" - cifmw_run_tempest: true cifmw_run_test_role: test_operator cifmw_run_tests: true cifmw_test_operator_concurrency: 4