diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index f3cc44ac..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,69 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -## [in development] - -### Added - -- Dev CLI - Add command "go run ./main.go dev getImagesSecurityIssues" to ease getting a small report of HIGH - and CRITICAL Security issues reported by quay.io on container images used by the sf-operator. - -### Changed - -- Zookeeper version bumped to 3.8.4 -- The Operator handles only one Route resource as a 'gateway' pod dispatches incoming connections. - -### Deprecated -### Removed - -- The LogsServer CRD and controller. As there is no identified need for a proper CRD and Controller. - -### Fixed -### Security - -- UBI9/Zookeeper image rebuid to address reported security issues - -## [v0.0.27] - 2024-03-27 - -🐰🔔 Easter release 🐰🔔 - -### Added - -- "Debug" toggle for fluent bit sidecars -- A support for running zuul-executor component external to the cluster (see ADR#014). -- The standalone deployment mode exits 1 when the reconcile is not possible after 300 seconds -- A bundled YAML file containing information about container images used by the operator [images.yaml](./controllers/libs/base/static/images.yaml) - -### Changed - -- zookeeper: update liveness and readyness probes to only check SSL access and remove superfluous Service resource called - zookeeper-headless. -- nodepool: update version to 10.0.0 -- zuul: update version to 10.0.0 -- CLI: simplified `SF backup` options to streamline the backup process. - -### Deprecated -### Removed -### Fixed - -- nodepool-builder: fixed the log path configuration when using the fluent bit log forwarder, resulting in much less file access errors appearing in fluent bit logs. - -### Security - -## [v0.0.26] - 2024-03-08 - -### Added - -- CLI: Add the `SF backup` subcommand. This subcommand dumps a Software Factory's most important data for safekeeping. - -### Changed -### Deprecated -### Removed -### Fixed -### Security - -## [alpha] - not released - -- Initial alpha version. Please consult the commit log for detailed information. -- From now on all changes will be referenced into this changelog. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 120000 index 00000000..9d2965c7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1 @@ +doc/reference/CHANGELOG.md \ No newline at end of file diff --git a/cli/cmd/restore.go b/cli/cmd/restore.go index d7ff7f94..81293b09 100644 --- a/cli/cmd/restore.go +++ b/cli/cmd/restore.go @@ -32,7 +32,9 @@ import ( "github.com/spf13/cobra" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" ) @@ -162,6 +164,33 @@ func restoreZuul(ns string, backupDir string, kubeClientSet *kubernetes.Clientse } +func clearComponents(ns string, kubeContext string) { + ctrl.Log.Info("Removing components requiring a complete restart ...") + + env := cliutils.ENV{ + Cli: cliutils.CreateKubernetesClientOrDie(kubeContext), + Ctx: context.TODO(), + Ns: ns, + } + + for _, stsName := range []string{"zuul-scheduler", "zuul-executor", "zuul-merger", "nodepool-builder", "zookeeper"} { + cliutils.DeleteOrDie(&env, &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: stsName, + Namespace: env.Ns, + }, + }) + } + for _, depName := range []string{"zuul-web", "nodepool-launcher"} { + cliutils.DeleteOrDie(&env, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: depName, + Namespace: env.Ns, + }, + }) + } +} + func restoreCmd(kmd *cobra.Command, args []string) { // NOTE: Solution for restoring DB and Zuul require kubectl binary to be installed and configured .kube/config @@ -191,6 +220,7 @@ func restoreCmd(kmd *cobra.Command, args []string) { restoreZuul(ns, backupDir, kubeClientSet, kubeContext) restoreSecret(ns, backupDir, kubeContext) restoreDB(ns, backupDir, kubeClientSet, kubeContext) + clearComponents(ns, kubeContext) } diff --git a/controllers/config.go b/controllers/config.go index d19b2e0d..2ef1533b 100644 --- a/controllers/config.go +++ b/controllers/config.go @@ -166,6 +166,10 @@ func (r *SFController) InstallTooling() { func (r *SFController) SetupConfigJob() bool { + // Get the resource version of the keystore password + zkp := apiv1.Secret{} + r.GetM("zuul-keystore-password", &zkp) + // This ensure we trigger the base secret creation job when the setting change extraSettingsChecksum := "ns" if r.cr.Spec.ConfigRepositoryLocation.ClusterAPIURL != "" || r.cr.Spec.ConfigRepositoryLocation.LogserverHost != "" { @@ -179,7 +183,7 @@ func (r *SFController) SetupConfigJob() bool { cmName = "zs-internal-tenant-reconfigure" zsInternalTenantReconfigure apiv1.ConfigMap configHash = utils.Checksum([]byte(r.MkPreInitScript())) - internalTenantSecretsVersion = "1" + "-" + extraSettingsChecksum + internalTenantSecretsVersion = "1" + "-" + zkp.ResourceVersion + "-" + extraSettingsChecksum needReconfigureTenant = false needCMUpdate = false ) diff --git a/doc/README.md b/doc/README.md index e3860e4e..b2a92050 100644 --- a/doc/README.md +++ b/doc/README.md @@ -45,10 +45,10 @@ The current project status is: **Alpha - NOT PRODUCTION READY** * Level 2 - Seamless upgrades - **2/2** - Operator: ✅ - Operands: ✅ -* Level 3 - Full Lifecycle - **1/5** +* Level 3 - Full Lifecycle - **3/5** - SF 3.8.x migration ❌ - - Backup: ❌ - - Restore: ❌ + - Backup: ✅ + - Restore: ✅ - Rolling deployments: ❌ - Reconfiguration: ✅ * Level 4 - Deep Insights - **1/3** diff --git a/doc/deployment/backup-restore.md b/doc/deployment/backup-restore.md new file mode 100644 index 00000000..7880541a --- /dev/null +++ b/doc/deployment/backup-restore.md @@ -0,0 +1,20 @@ +# Backup and Restore + +The sf-operator CLI provides commands to perform a backup and a restore of a deployment managed by the sf-operator. + +The [backup command](../reference/cli/index.md#backup) can be run periodically to perform a backup of a Software Factory deployment. +The command should be couple with a proper backup system to safely store the backuped data. + +Restoring a backup must be done via the [restore command](../reference/cli/index.md#restore) and by following a specific process: + +1. Deploy a Software Factory CR resource without `ConfigLocation` and Zuul connections definition. +2. Restore the backup with the `restore` command +3. Apply your final Software Factory CR + +## The backup archive + +The archive contains: + +- Some k8s Secret resources (like the Zuul Keystore Secret and Zuul SSH private key Secret) +- The Zuul SQL database content (history of builds) +- The Zuul projects' private keys (the keys stored into Zookeeper and used to encrypt/decrypt in-repo Zuul Secrets) \ No newline at end of file diff --git a/doc/deployment/index.md b/doc/deployment/index.md index 07e52c5a..37eee8c9 100644 --- a/doc/deployment/index.md +++ b/doc/deployment/index.md @@ -20,5 +20,6 @@ and managing a Software Factory Custom Resource through SF-Operator. 1. [Add corporate CA certificates into the CA trust chain](./corporate-certificates.md) 1. [Monitoring](./monitoring.md) 1. [Logging](./logging.md) +1. [Backup and Restore](./backup-restore.md) 1. [Deleting a deployment](./delete.md) 1. [Custom Resource Definitions reference](./crds.md) diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md deleted file mode 120000 index 699cc9e7..00000000 --- a/doc/reference/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/doc/reference/CHANGELOG.md b/doc/reference/CHANGELOG.md new file mode 100644 index 00000000..e23880d3 --- /dev/null +++ b/doc/reference/CHANGELOG.md @@ -0,0 +1,70 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [in development] + +### Added + +- CLI: restore command and documentation. +- Dev CLI - Add command "go run ./main.go dev getImagesSecurityIssues" to ease getting a small report of HIGH + and CRITICAL Security issues reported by quay.io on container images used by the sf-operator. + +### Changed + +- Zookeeper version bumped to 3.8.4 +- The Operator handles only one Route resource as a 'gateway' pod dispatches incoming connections. + +### Deprecated +### Removed + +- The LogsServer CRD and controller. As there is no identified need for a proper CRD and Controller. + +### Fixed +### Security + +- UBI9/Zookeeper image rebuid to address reported security issues + +## [v0.0.27] - 2024-03-27 + +🐰🔔 Easter release 🐰🔔 + +### Added + +- "Debug" toggle for fluent bit sidecars +- A support for running zuul-executor component external to the cluster (see ADR#014). +- The standalone deployment mode exits 1 when the reconcile is not possible after 300 seconds +- A bundled YAML file containing information about container images used by the operator `controllers/libs/base/static/images.yaml` + +### Changed + +- zookeeper: update liveness and readyness probes to only check SSL access and remove superfluous Service resource called + zookeeper-headless. +- nodepool: update version to 10.0.0 +- zuul: update version to 10.0.0 +- CLI: simplified `SF backup` options to streamline the backup process. + +### Deprecated +### Removed +### Fixed + +- nodepool-builder: fixed the log path configuration when using the fluent bit log forwarder, resulting in much less file access errors appearing in fluent bit logs. + +### Security + +## [v0.0.26] - 2024-03-08 + +### Added + +- CLI: Add the `SF backup` subcommand. This subcommand dumps a Software Factory's most important data for safekeeping. + +### Changed +### Deprecated +### Removed +### Fixed +### Security + +## [alpha] - not released + +- Initial alpha version. Please consult the commit log for detailed information. +- From now on all changes will be referenced into this changelog. diff --git a/doc/reference/cli/index.md b/doc/reference/cli/index.md index 5394e3b7..34a4a650 100644 --- a/doc/reference/cli/index.md +++ b/doc/reference/cli/index.md @@ -4,11 +4,11 @@ We provide a command to perform various actions related to the management of Sof deployments, beyond what can be defined in a custom resource manifest. -1. [Installing the CLI](#installing-the-cli) -1. [Global Flags](#global-flags) -1. [Configuration File](#configuration-file) -1. [Subcommands](#subcommands) - 1. [Dev](#dev) +- [Installing the CLI](#installing-the-cli) +- [Global Flags](#global-flags) +- [Configuration File](#configuration-file) +- [Subcommands](#subcommands) + - [Dev](#dev) - [cloneAsAdmin](#cloneasadmin) - [create demo-env](#create-demo-env) - [create gerrit](#create-gerrit) @@ -17,20 +17,20 @@ deployments, beyond what can be defined in a custom resource manifest. - [run-tests](#run-tests) - [wipe gerrit](#wipe-gerrit) - [getImagesSecurityIssues](#getimagessecurityissues) - 1. [Init](#init) - 1. [Nodepool](#nodepool) + - [Init](#init) + - [Nodepool](#nodepool) - [configure providers-secrets](#configure-providers-secrets) - [create openshiftpods-namespace](#create-openshiftpods-namespace) - [get builder-ssh-key](#get-builder-ssh-key) - [get providers-secrets](#get-providers-secrets) - 1. [Operator](#operator) - 1. [SF](#sf) - 1. [backup](#backup) - 1. [bootstrap-tenant](#bootstrap-tenant) - 1. [configure TLS](#configure-tls) - 1. [restore](#restore) - 1. [wipe](#wipe) - 1. [Zuul](#zuul) + - [Operator](#operator) + - [SF](#sf) + - [backup](#backup) + - [bootstrap-tenant](#bootstrap-tenant) + - [configure TLS](#configure-tls) + - [restore](#restore) + - [wipe](#wipe) + - [Zuul](#zuul) - [create auth-token](#create-auth-token) - [create client-config](#create-client-config) @@ -418,14 +418,9 @@ The following subcommands can be used to manage a Software Factory deployment an #### backup -The `backup` subcommand lets you dump a Software Factory's most important files for safekeeping, -most important files, such as: +The `backup` subcommand lets you dump a Software Factory's most important files for safekeeping. -- MariaDB Zuul database copy -- secrets backup -- Zuul project private keys - -To create a backup located in `/tmp/backup` directory of all important objects, run the following command: +To create a backup located in `/tmp/backup` directory, run the following command: ```sh sf-operator SF backup --namespace sf --backup_dir /tmp/backup @@ -437,6 +432,14 @@ Flags: |----------|------|-------|----|----| | --backup_dir | string | The path to the backup directory | no | - | +The backup is composed of: + +- some relevant `Secrets` located in the deployment's namespace +- the Zuul's SQL database +- the Zuul's project's keys as exported by [zuul-admin export-keys](https://zuul-ci.org/docs/zuul/latest/client.html#export-keys) + +The backup directory content could be compressed and stored safely in a backup system. + #### bootstrap-tenant Initialize a Zuul tenant's config repository with boilerplate code that define standard pipelines: @@ -480,13 +483,7 @@ Flags: !!! warning The command requires to to have `kubectl` binary installed in the system -The `restore` subcommand lets you restore: - -- Secrets -- MariaDB Zuul database -- Zuul project private keys - -that has been done by the `backup` command. +The `restore` subcommand lets you restore a backup created with the `backup` command. For example: @@ -500,6 +497,7 @@ Available flags: |----------|------|-------|----|----| | --backup_dir | string | The path to the backup directory to restore | yes | - | + #### wipe The `wipe` subcommand can be used to remove all Software Factory instances in the provided namespace, diff --git a/playbooks/files/sf-minimal.yaml b/playbooks/files/sf-minimal.yaml new file mode 100644 index 00000000..1ce937d8 --- /dev/null +++ b/playbooks/files/sf-minimal.yaml @@ -0,0 +1,6 @@ +apiVersion: sf.softwarefactory-project.io/v1 +kind: SoftwareFactory +metadata: + name: my-sf +spec: + fqdn: "sfop.me" diff --git a/roles/health-check/backup/defaults/main.yaml b/roles/health-check/backup/defaults/main.yaml deleted file mode 100644 index c36b9935..00000000 --- a/roles/health-check/backup/defaults/main.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -backup_dir: "{{ ansible_user_dir }}/backup" -namespace: sf diff --git a/roles/health-check/backup/tasks/check_dir.yaml b/roles/health-check/backup/tasks/check_dir.yaml deleted file mode 100644 index 371c11ba..00000000 --- a/roles/health-check/backup/tasks/check_dir.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: "Check if backup directory is not empty - {{ backup_dir }}/{{ backup_subdir }}" - ansible.builtin.find: - paths: "{{ backup_dir }}/{{ backup_subdir }}" - register: _backup_files - -- name: "Ensure, that the backup sub directory is not empty - {{ backup_dir }}/{{ backup_subdir }}" - when: _backup_files.matched == 0 - ansible.builtin.fail: - msg: "The directory is empty {{ backup_dir }}/{{ backup_subdir }}. Something is wrong with backup tool!" diff --git a/roles/health-check/backup/tasks/main.yaml b/roles/health-check/backup/tasks/main.yaml deleted file mode 100644 index a624a1d3..00000000 --- a/roles/health-check/backup/tasks/main.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Run sf-operator backup - ansible.builtin.command: | - go run main.go SF backup --backup_dir {{ backup_dir }} {{ cli_global_flags }} - args: - chdir: "{{ zuul.project.src_dir }}" - -- name: Ensure directories are not empty - ansible.builtin.include_tasks: - file: check_dir.yaml - loop: - - secrets - - mariadb - - zuul - loop_control: - loop_var: backup_subdir - -- name: Compare secrets - ansible.builtin.include_tasks: - file: secrets.yaml - loop: - - name: zuul-keystore-password - data_key: zuul-keystore-password - loop_control: - loop_var: secret_name - -- name: Check SQL backup - ansible.builtin.include_tasks: - file: sql.yaml - -- name: Check Zuul backup - ansible.builtin.include_tasks: - file: zuul.yaml diff --git a/roles/health-check/backup/tasks/secrets.yaml b/roles/health-check/backup/tasks/secrets.yaml deleted file mode 100644 index c8342682..00000000 --- a/roles/health-check/backup/tasks/secrets.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: "Get password from the secret - {{ secret_name.name }}" - ansible.builtin.shell: | - kubectl -n {{ namespace }} get secret {{ secret_name.name }} -o go-template --template {% raw %}'{{.data}}' {% endraw %} | cut -f2 -d':' - register: _b64_pass - -- name: "Get password from a backup file - {{ secret_name.name }}" - ansible.builtin.shell: | - grep '{{ secret_name.data_key }}:' {{ backup_dir }}/secrets/{{ secret_name.name }}.yaml | awk '{print $2}' - register: _file_pass - -- name: Ensure that password from backup and in Kubernetes are same - ansible.builtin.assert: - that: - - _b64_pass.stdout | b64decode == _file_pass.stdout diff --git a/roles/health-check/backup/tasks/sql.yaml b/roles/health-check/backup/tasks/sql.yaml deleted file mode 100644 index 6150add5..00000000 --- a/roles/health-check/backup/tasks/sql.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Check if sql backup contains CREATE DATABASE phrase - ansible.builtin.find: - paths: "{{ backup_dir }}" - patterns: "*.sql" - contains: 'CREATE DATABASE .*zuul.*utf8' - read_whole_file: true - recurse: true - register: _sql_output - -- name: Ensure CREATE DATABASE for zuul has been called once - ansible.builtin.assert: - that: - - _sql_output.files | length == 1 diff --git a/roles/health-check/backup/tasks/zuul.yaml b/roles/health-check/backup/tasks/zuul.yaml deleted file mode 100644 index 37fa40cb..00000000 --- a/roles/health-check/backup/tasks/zuul.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Count BEGIN RSA PRIVATE KEY fromz Zuul backup - ansible.builtin.shell: | - tr '\\n' '\n' < {{ backup_dir }}/zuul/zuul.keys | grep "BEGIN RSA PRIVATE KEY" | wc -l - register: _zuul_file_rsa_count - -- name: Ensure each Zuul project has own private key - ansible.builtin.assert: - that: - - _zuul_file_rsa_count.stdout | int > 0 diff --git a/roles/health-check/restore/defaults/main.yaml b/roles/health-check/restore/defaults/main.yaml deleted file mode 100644 index 236bf8a2..00000000 --- a/roles/health-check/restore/defaults/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -namespace: sf -backup_dir: "{{ ansible_user_dir }}/backup" -restore_dir: "{{ ansible_user_dir }}/restore" -restore_secret_name: zuul-keystore-password -restore_zuul_path: "{{ backup_dir }}/zuul/zuul.keys" diff --git a/roles/health-check/restore/tasks/change.yaml b/roles/health-check/restore/tasks/change.yaml deleted file mode 100644 index 4d4ed1c5..00000000 --- a/roles/health-check/restore/tasks/change.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -- name: Set facts for secret manipulation - ansible.builtin.set_fact: - test_secret_key: "{{ secret_before.data | first }}" - test_secret_value: "{{ 'restoreTest' | b64encode }}" - -- name: "Replace secret with random data - {{ restore_secret_name }}" - kubernetes.core.k8s: - state: patched - kind: Secret - name: "{{ secret_before.metadata.name }}" - namespace: "{{ namespace }}" - definition: - data: - zuul-keystore-password: "{{ test_secret_value }}" - -- name: "Get secret result {{ restore_secret_name }} in namespace sf after change" - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "{{ restore_secret_name }}" - namespace: "{{ namespace }}" - register: secret_result - -- name: Get base64 for fake data value - ansible.builtin.set_fact: - secret_fake: "{{ secret_result.resources[0] }}" - -- name: Assert that secrets b64 are different - ansible.builtin.assert: - that: - - secret_before.data != secret_fake.data - -- name: Create test table in zuul db - zuul_restore_test - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: mariadb-0 - command: > - mysql -h 0 -u root - -p{{ db_root_pass }} - -e "USE zuul; CREATE TABLE zuul_restore_test (ID int);" - -- name: Get available tables - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: mariadb-0 - command: > - mysql -h 0 -u root - -p{{ db_root_pass }} - -e "USE zuul; SHOW TABLES;" - register: _show_tables - -- name: Ensure that there is zuul_restore_test table - ansible.builtin.assert: - that: - - "'zuul_restore_test' in _show_tables.stdout" - -- name: Remove the project key from Zuul - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: zuul-scheduler-0 - command: > - zuul-admin delete-keys gerrit demo-tenant-config - -- name: Export current Zuul keys after deletion - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: zuul-scheduler-0 - command: > - zuul-admin export-keys /tmp/removed-key - -- name: Check if the demo-tenant-config is in new exported file - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: zuul-scheduler-0 - command: > - grep -c demo-tenant-config /tmp/removed-key - register: _count_project_remove_key - failed_when: _count_project_remove_key.rc not in [0,1] - -- name: Assert that the demo-tenant-config has been removed - ansible.builtin.assert: - that: - - _count_project_remove_key.stdout | int == 0 diff --git a/roles/health-check/restore/tasks/main.yaml b/roles/health-check/restore/tasks/main.yaml deleted file mode 100644 index c8d9b8cc..00000000 --- a/roles/health-check/restore/tasks/main.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Pre check restore - ansible.builtin.include_tasks: pre.yaml - -- name: Change content - ansible.builtin.include_tasks: change.yaml - -- name: Post check restore - verify - ansible.builtin.include_tasks: post.yaml diff --git a/roles/health-check/restore/tasks/post.yaml b/roles/health-check/restore/tasks/post.yaml deleted file mode 100644 index 5f5c4bf7..00000000 --- a/roles/health-check/restore/tasks/post.yaml +++ /dev/null @@ -1,58 +0,0 @@ ---- -- name: Run sf-operator restore - ansible.builtin.command: | - go run main.go SF restore --backup_dir {{ backup_dir }} {{ cli_global_flags }} - args: - chdir: "{{ zuul.project.src_dir }}" - -- name: Get secret test in namespace sf after restore - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "{{ restore_secret_name }}" - namespace: "{{ namespace }}" - register: secret_result - -- name: Get base64 for restore data value - ansible.builtin.set_fact: - secret_restore: "{{ secret_result.resources[0] }}" - -- name: Assert that secrets b64 after restore are same - ansible.builtin.assert: - that: - - secret_before.data == secret_restore.data - -- name: Get available tables - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: mariadb-0 - command: > - mysql -h 0 -u root - -p{{ db_root_pass }} - -e "USE zuul; SHOW TABLES;" - register: _show_tables - -- name: Ensure that there is no Zuul DB - ansible.builtin.assert: - that: - - "'zuul_restore_test' not in _show_tables.stdout" - -- name: Export new Zuul keys in the pod after restore - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: zuul-scheduler-0 - command: > - zuul-admin export-keys /tmp/restored-key - -- name: Check if the demo-tenant-config is in new exported file - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: zuul-scheduler-0 - command: > - grep -c demo-tenant-config /tmp/restored-key - register: _count_project_restore_key - -- name: Assert that the demo-tenant-config has been removed - ansible.builtin.assert: - that: - - _count_project_restore_key.stdout | int == 1 diff --git a/roles/health-check/restore/tasks/pre.yaml b/roles/health-check/restore/tasks/pre.yaml deleted file mode 100644 index b8d6184b..00000000 --- a/roles/health-check/restore/tasks/pre.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# This playbook will just check if base64 encode is same as before -- name: Create working dir - ansible.builtin.file: - path: "{{ restore_dir }}" - state: directory - -- name: "Get secret {{ restore_secret_name }} content in namespace {{ namespace }}" - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "{{ restore_secret_name }}" - namespace: "{{ namespace }}" - register: secret_result - -- name: Get base64 for current secret - ansible.builtin.set_fact: - secret_before: "{{ secret_result.resources[0] }}" - -- name: Get MariaDB root password - kubernetes.core.k8s_exec: - namespace: "{{ namespace }}" - pod: mariadb-0 - command: | - bash -c "env | grep MYSQL_ROOT_PASSWORD | cut -f2 -d'=' | tr -d '[:space:]'" - register: _db_pass - -- ansible.builtin.set_fact: - db_root_pass: "{{ _db_pass.stdout }}" - -# zuul-admin delete-keys gerrit demo-tenant-config -- name: Check if the demo-tenant-config is in backup - ansible.builtin.shell: | - grep -c demo-tenant-config {{ restore_zuul_path }} - register: _count_project - -- name: Assert that the demo-tenant-config exists - ansible.builtin.assert: - that: - - _count_project.stdout | int > 0 diff --git a/roles/health-check/test-backup-restore/defaults/main.yaml b/roles/health-check/test-backup-restore/defaults/main.yaml new file mode 100644 index 00000000..689a5424 --- /dev/null +++ b/roles/health-check/test-backup-restore/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +backup_dir: "/tmp/sf-backup" diff --git a/roles/health-check/test-backup-restore/tasks/main.yaml b/roles/health-check/test-backup-restore/tasks/main.yaml new file mode 100644 index 00000000..46a9ebe6 --- /dev/null +++ b/roles/health-check/test-backup-restore/tasks/main.yaml @@ -0,0 +1,107 @@ +--- +# This test case assumes that test role 'config-update-zuul' run once previouly in the functional tests case +# This test case validate the backup / restore process + +# 1. We backup to current deployment +- name: Backup the Software Factory deployment + ansible.builtin.command: | + go run main.go {{ cli_global_flags }} SF backup --backup_dir {{ backup_dir }} + args: + chdir: "{{ zuul.project.src_dir }}" + +# 2. We gather the last Zuul buildset info +- name: Fetch the last build ID reported by Zuul web + ansible.builtin.uri: + url: "https://{{ zuul_endpoint }}/api/tenant/internal/buildsets?skip=0&limit=1" + method: GET + return_content: true + body_format: json + validate_certs: "{{ validate_certs }}" + register: last_buildset + until: + - "'json' in last_buildset" + retries: "{{ zuul_api_retries }}" + delay: "{{ zuul_api_delay }}" + +# 3. We wipe the Software Factory deployment to simulate a disaster +- name: Wipe Software Factory deployment + ansible.builtin.command: | + go run main.go {{ cli_global_flags }} SF wipe --rm-data + args: + chdir: "{{ zuul.project.src_dir }}" + +# 4. We spawn a new minimal Software Factory +- name: Make a new minimal Software Factory deployment (standalone) + ansible.builtin.include_role: + name: run-operator-standalone + vars: + glue: false + cr_path: playbooks/files/sf-minimal.yaml + when: "{{ mode == 'standalone' }}" +- name: Make a new minimal Software Factory deployment (olm) + ansible.builtin.include_role: + name: apply-custom-resources + vars: + cr_path: playbooks/files/sf-minimal.yaml + when: "{{ mode == 'olm' }}" + +# 4 bis. For the paranoid - Check the builds list reported by Zuul is empty +- name: Check the builds list reported by Zuul is empty (after wipe and before restoring data) + ansible.builtin.uri: + url: "https://{{ zuul_endpoint }}/api/tenant/internal/buildsets?skip=0&limit=1" + method: GET + return_content: true + body_format: json + validate_certs: "{{ validate_certs }}" + register: _last_buildset + until: + - "'json' in last_buildset" + retries: "{{ zuul_api_retries }}" + delay: "{{ zuul_api_delay }}" +- name: Ensure no result in Zuul SQL buildsets + ansible.builtin.assert: + that: + - _last_buildset.json | length == 0 + +# 5. We restore the backup +- name: Restore backup of the Software Factory previous deployment + ansible.builtin.command: | + go run main.go {{ cli_global_flags }} SF restore --backup_dir {{ backup_dir }} + args: + chdir: "{{ zuul.project.src_dir }}" + +# 6. We re-enable the config location and other settings +- name: Apply the Software Factory deployment (standalone) + ansible.builtin.include_role: + name: run-operator-standalone + vars: + glue: false + when: "{{ mode == 'standalone' }}" + +- name: Apply the Software Factory deployment (olm) + ansible.builtin.include_role: + name: apply-custom-resources + when: "{{ mode == 'olm' }}" + +# 7. We ensure we recovered the Zuul SQL Database content +- name: Fetch the last build ID reported by Zuul web (after restore) + ansible.builtin.uri: + url: "https://{{ zuul_endpoint }}/api/tenant/internal/buildsets?skip=0&limit=1" + method: GET + return_content: true + body_format: json + validate_certs: "{{ validate_certs }}" + register: _last_buildset + until: + - "'json' in last_buildset" + retries: "{{ zuul_api_retries }}" + delay: "{{ zuul_api_delay }}" +- name: Ensure Zuul SQL DB well restored by checking last build ID + ansible.builtin.assert: + that: + - _last_buildset.json[0].uuid == last_buildset.json[0].uuid + +# 8. We run a zuul job workflow to ensure no failure +- name: Run a Zuul job workflow in order to validate the restored system + ansible.builtin.include_role: + name: health-check/config-update-zuul diff --git a/roles/run-tests/tasks/main.yaml b/roles/run-tests/tasks/main.yaml index 6bd751cd..c5c10e63 100644 --- a/roles/run-tests/tasks/main.yaml +++ b/roles/run-tests/tasks/main.yaml @@ -42,7 +42,6 @@ - name: zuul-components - name: test-external-ze - name: test-log-forwarding - - name: backup - - name: restore + - name: test-backup-restore loop_control: loop_var: role