diff --git a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml index 92a234c078..7e003bc941 100644 --- a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml +++ b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml @@ -6193,6 +6193,10 @@ spec: type: boolean extensions: properties: + custom: + items: + type: string + type: array pg_tde: properties: enabled: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 19dbe090d8..1b9df82469 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -27103,6 +27103,10 @@ spec: type: boolean extensions: properties: + custom: + items: + type: string + type: array pg_tde: properties: enabled: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index aac5a9534a..c195cde670 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -27103,6 +27103,10 @@ spec: type: boolean extensions: properties: + custom: + items: + type: string + type: array pg_tde: properties: enabled: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index e881798213..3704afcbb6 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -27103,6 +27103,10 @@ spec: type: boolean extensions: properties: + custom: + items: + type: string + type: array pg_tde: properties: enabled: diff --git a/e2e-tests/functions b/e2e-tests/functions index abf2aaa4c9..73642e7d21 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -347,7 +347,7 @@ function deploy_s3_secrets() { kubectl -n "$NAMESPACE" create secret generic "${test_name}-pgbackrest-secrets" "${secret_args[@]}" case $test_name in - "custom-extensions" | "builtin-extensions" | major-upgrade*) + "custom-extensions" | "builtin-extensions" | "operator-upgrade-extensions" | major-upgrade*) kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/cloud-secret.yml" kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/minio-secret.yml" ;; @@ -2298,6 +2298,119 @@ function verify_k8s_nodes_version() { done } +# Save sts update revisions + deployment revisions of the cluster workloads. +snapshot_workload_revisions() { + local cluster="$1" out="$2" + kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster="$cluster" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.status.updateRevision}{"\n"}{end}' >"$out" + kubectl -n "$NAMESPACE" get deploy -l postgres-operator.crunchydata.com/cluster="$cluster" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' >>"$out" +} + +# Fail if any sts/deploy revision changed (operator-driven rollout). +# Container restarts are tolerated only on pods whose node was evicted. +assert_no_operator_rollout() { + local cluster="$1" baseline="$2" current="${TEMP_DIR}/revisions.current" + snapshot_workload_revisions "$cluster" "$current" + if ! diff -u "$baseline" "$current"; then + echo "ERROR: workload revisions changed - operator triggered a rollout" + return 1 + fi + local restarted + restarted=$(kubectl -n "$NAMESPACE" get pods -l postgres-operator.crunchydata.com/cluster="$cluster" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.status.containerStatuses[*].restartCount}{"\n"}{end}' \ + | grep -Ev '=(0 ?)*0?$' | cut -d= -f1 || true) + local pod + for pod in $restarted; do + if ! kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pod" \ + -o jsonpath='{range .items[*]}{.reason}{"\n"}{end}' | grep -qE 'TaintManagerEviction|NodeNotReady|Preempt'; then + echo "ERROR: pod $pod restarted without node eviction" + return 1 + fi + done + return 0 +} + +# sha256 of tls.crt of every TLS-bearing secret of the cluster. +snapshot_tls_fingerprints() { + local cluster="$1" out="$2" s crt + : >"$out" + for s in $(kubectl -n "$NAMESPACE" get secrets -l postgres-operator.crunchydata.com/cluster="$cluster" -o name); do + crt=$(kubectl -n "$NAMESPACE" get "$s" -o jsonpath='{.data.tls\.crt}') + if [[ -n $crt ]]; then + echo "${s}=$(echo "$crt" | sha256sum | awk '{print $1}')" >>"$out" + fi + done + sort -o "$out" "$out" +} + +assert_tls_unchanged() { + local current="${TEMP_DIR}/tls.current" + snapshot_tls_fingerprints "$1" "$current" + if ! diff -u "$2" "$current"; then + echo "ERROR: TLS certificates were re-issued" + return 1 + fi +} + +# Background INSERT loop through pgbouncer; pid stored in TEMP_DIR. +start_heartbeat() { + local cluster="$1" + local uri="postgres:$(get_psql_user_pass ${cluster}-pguser-postgres)@$(get_psql_user_host ${cluster}-pguser-postgres)" + run_psql_local 'CREATE TABLE IF NOT EXISTS heartbeat (ts timestamptz)' "$uri" + ( + set +o xtrace + while true; do + run_psql_local 'INSERT INTO heartbeat VALUES (now())' "$uri" || true + sleep 2 + done + ) >/dev/null 2>&1 & + echo $! >"${TEMP_DIR}/heartbeat.pid" +} + +assert_heartbeat_gap() { + local cluster="$1" max_gap="$2" + kill "$(cat "${TEMP_DIR}/heartbeat.pid")" 2>/dev/null || true + local uri="postgres:$(get_psql_user_pass ${cluster}-pguser-postgres)@$(get_psql_user_host ${cluster}-pguser-postgres)" + local gap + gap=$(run_psql_local 'SELECT COALESCE(ceil(EXTRACT(EPOCH FROM max(ts - prev)))::int,0) FROM (SELECT ts, lag(ts) OVER (ORDER BY ts) AS prev FROM heartbeat) g' "$uri" | tr -d '[:space:]') + echo "heartbeat max gap: ${gap}s (limit ${max_gap}s)" + if [[ -z "$gap" || "$gap" -gt "$max_gap" ]]; then + echo "ERROR: heartbeat gap '${gap}' exceeds ${max_gap}s" + return 1 + fi +} + +# Scan operator + postgres logs and Warning events since a timestamp. +check_logs_for_errors() { + local cluster="$1" since="$2" allow="$3" fail=0 pod f + local op_ns="${OPERATOR_NS:-$NAMESPACE}" + rm -f "${TEMP_DIR}"/log_errors_*.txt + # the operator logs JSON only with LOG_STRUCTURED=true, deploy/operator.yaml + # sets it to false, so match the console encoder format as well + kubectl -n "$op_ns" logs deployment/percona-postgresql-operator --since-time="$since" 2>/dev/null \ + | grep -E '"level":"error"|[[:space:]]ERROR[[:space:]]|panic:' | grep -Evf "$allow" >"${TEMP_DIR}/log_errors_operator.txt" || true + for pod in $(kubectl -n "$NAMESPACE" get pods -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres -o name); do + kubectl -n "$NAMESPACE" logs "$pod" -c database --since-time="$since" 2>/dev/null \ + | grep -E 'FATAL|PANIC' | grep -Evf "$allow" >>"${TEMP_DIR}/log_errors_postgres.txt" || true + done + # event messages can be multi-line (pgBackRest output): fold every event back + # into a single line, otherwise the continuation lines pass both the + # timestamp filter and the allowlist, which only matches the first line + kubectl -n "$NAMESPACE" get events --field-selector type=Warning \ + -o jsonpath='{range .items[*]}{.lastTimestamp} {.involvedObject.name} {.reason} {.message}{"\n"}{end}' 2>/dev/null \ + | awk '/^[0-9]{4}-[0-9]{2}-[0-9]{2}T/ {if (rec != "") print rec; rec = $0; next} {rec = rec " " $0} END {if (rec != "") print rec}' \ + | awk -v s="$since" '$1 >= s' | grep -Evf "$allow" >"${TEMP_DIR}/log_errors_events.txt" || true + for f in operator postgres events; do + if [[ -s "${TEMP_DIR}/log_errors_${f}.txt" ]]; then + echo "=== unexpected ${f} errors since ${since} ===" + cat "${TEMP_DIR}/log_errors_${f}.txt" + fail=1 + fi + done + return $fail +} + function vault_tls() { local name=${1:-vault-service} local tmp_dir=$2 diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 91d87b8de7..fb99612750 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -22,6 +22,8 @@ ldap ldap-tls logcollection monitoring +operator-upgrade-cert-manager +operator-upgrade-extensions one-pod repo-host-autogrow operator-self-healing diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index 0a3a2f09d4..1f92eb63e3 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -27,6 +27,8 @@ ldap ldap-tls logcollection monitoring +operator-upgrade-cert-manager +operator-upgrade-extensions one-pod repo-host-autogrow operator-self-healing diff --git a/e2e-tests/tests/custom-extensions/11-assert.yaml b/e2e-tests/tests/custom-extensions/11-assert.yaml index d655a322aa..c93d54b5dd 100644 --- a/e2e-tests/tests/custom-extensions/11-assert.yaml +++ b/e2e-tests/tests/custom-extensions/11-assert.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 120 +timeout: 240 --- kind: StatefulSet apiVersion: apps/v1 diff --git a/e2e-tests/tests/custom-extensions/17-assert.yaml b/e2e-tests/tests/custom-extensions/17-assert.yaml new file mode 100644 index 0000000000..7b4e037224 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/17-assert.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# adding pg_cron to the custom list and to shared_preload_libraries restarts +# every instance pod +timeout: 420 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: custom-extensions +status: + installedCustomExtensions: + - pg_stat_kcache + - pg_cron + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml b/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml new file mode 100644 index 0000000000..debc2efd72 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml @@ -0,0 +1,20 @@ +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: custom-extensions +spec: + # pg_cron is installed as a custom extension here so the next steps can move + # it to spec.extensions.pg_cron: it is the only extension that is both + # available as a custom archive and managed as a builtin one, and it owns + # data (the cron.job rows) that a wrong drop would destroy + extensions: + custom: + - name: pg_stat_kcache + version: 2.3.2 + - name: pg_cron + version: 1.6.7 + patroni: + dynamicConfiguration: + postgresql: + parameters: + shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron diff --git a/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml b/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml new file mode 100644 index 0000000000..c71fff4342 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/18-restart-cluster.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # pg_cron was added to shared_preload_libraries in the previous step and + # postgres only loads it on start, so the pods have to be restarted before + # the extension can be created + wait_cluster_consistency custom-extensions + + restart_pg_pods custom-extensions instance1 + timeout: 360 diff --git a/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml b/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml new file mode 100644 index 0000000000..26cee3cc53 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)" + + # cron.database_name only exists once the pg_cron library is loaded, so it + # tells us the restart of the previous step actually picked up the new + # shared_preload_libraries. CREATE EXTENSION fails without it. + for i in $(seq 1 30); do + if run_psql_local '\c postgres \\\ SHOW cron.database_name' "$uri" >/dev/null 2>&1; then + break + fi + sleep 5 + done + run_psql_local '\c postgres \\\ SHOW cron.database_name' "$uri" \ + || { echo "ERROR: pg_cron library is not loaded, shared_preload_libraries did not take effect"; exit 1; } + + run_psql_local '\c postgres \\\ CREATE EXTENSION pg_cron' "$uri" + + # a job in cron.job is the data a user loses if pg_cron gets dropped + run_psql_local '\c postgres \\\ SELECT cron.schedule($$migration-tick$$,$$* * * * *$$,$$SELECT 1$$)' "$uri" + + jobs=$(run_psql_local '\c postgres \\\ SELECT count(*) FROM cron.job WHERE jobname = $$migration-tick$$' "$uri" | tr -d '[:space:]') + [[ "$jobs" == "1" ]] || { echo "ERROR: failed to schedule the pg_cron job"; exit 1; } diff --git a/e2e-tests/tests/custom-extensions/20-assert.yaml b/e2e-tests/tests/custom-extensions/20-assert.yaml new file mode 100644 index 0000000000..6b02c2cdb2 --- /dev/null +++ b/e2e-tests/tests/custom-extensions/20-assert.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +# removing pg_cron from the custom list changes INSTALL_EXTENSIONS and restarts +# every instance pod +timeout: 600 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: custom-extensions +status: + # pg_cron is gone from the custom list, it is managed as a builtin now + installedCustomExtensions: + - pg_stat_kcache + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml b/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml new file mode 100644 index 0000000000..68ad48930e --- /dev/null +++ b/e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml @@ -0,0 +1,20 @@ +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: custom-extensions +spec: + # pg_cron moves from a custom extension to a builtin one: the operator takes + # over its lifecycle, but the extension and its data must survive the move. + # Dropping it here used to be the default behavior, because the extension + # disappears from the custom list. + extensions: + pg_cron: + enabled: true + custom: + - name: pg_stat_kcache + version: 2.3.2 + patroni: + dynamicConfiguration: + postgresql: + parameters: + shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron diff --git a/e2e-tests/tests/custom-extensions/21-check-migration.yaml b/e2e-tests/tests/custom-extensions/21-check-migration.yaml new file mode 100644 index 0000000000..b2e751301a --- /dev/null +++ b/e2e-tests/tests/custom-extensions/21-check-migration.yaml @@ -0,0 +1,20 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)" + + # moving an extension from spec.extensions.custom to a builtin flag must + # not drop it: the operator removes it from the custom list, but the + # builtin flag is enabled, so the extension and its data have to stay + installed=$(run_psql_local '\c postgres \\\ SELECT extname FROM pg_extension WHERE extname = $$pg_cron$$' "$uri" | tr -d '[:space:]') + [[ "$installed" == "pg_cron" ]] || { echo "ERROR: pg_cron was dropped by the custom to builtin migration"; exit 1; } + + jobs=$(run_psql_local '\c postgres \\\ SELECT count(*) FROM cron.job WHERE jobname = $$migration-tick$$' "$uri" | tr -d '[:space:]') + [[ "$jobs" == "1" ]] || { echo "ERROR: cron.job data lost in the custom to builtin migration"; exit 1; } diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/00-assert.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/00-assert.yaml new file mode 100644 index 0000000000..098d743e4c --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-and-minio +timeout: 180 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 + - script: kubectl assert exist-enhanced deployment minio-service -n ${NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/00-deploy-operator.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/00-deploy-operator.yaml new file mode 100644 index 0000000000..72826d35af --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/00-deploy-operator.yaml @@ -0,0 +1,43 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + destroy_cert_manager + + git_tag="v$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" + deploy_operator_gh $git_tag + deploy_client + + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/minio-secret.yml" + + access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret \ + -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" + secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret \ + -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" + + openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ + -keyout "${TEMP_DIR}/minio.key" \ + -out "${TEMP_DIR}/minio.crt" \ + -subj "/CN=minio-service" + + kubectl -n "${NAMESPACE}" create secret generic minio-tls \ + --from-file=public.crt="${TEMP_DIR}/minio.crt" \ + --from-file=private.key="${TEMP_DIR}/minio.key" + + deploy_minio minio-service minio-tls + + cat > "${TEMP_DIR}/pgbackrest-minio.ini" << INI + [global] + repo2-s3-key=${access_key} + repo2-s3-key-secret=${secret_key} + INI + + kubectl -n "${NAMESPACE}" create secret generic ${test_name}-pgbackrest-secrets \ + --from-file=cloud.conf="${TEMP_DIR}/pgbackrest-minio.ini" diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/01-assert.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/01-assert.yaml new file mode 100644 index 0000000000..60ad2f9484 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/01-assert.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: operator-upgrade-cert-manager +status: + state: ready + postgres: + ready: 3 + size: 3 + pgbouncer: + ready: 3 + size: 3 diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml new file mode 100644 index 0000000000..b7e8b5249f --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/01-create-cluster.yaml @@ -0,0 +1,49 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # plain RELEASED cr.yaml: no custom TLS -> secrets come from the operator internal PKI + lower_ver="$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" + curl -s "https://raw.githubusercontent.com/percona/percona-postgresql-operator/v${lower_ver}/deploy/cr.yaml" >"${TEMP_DIR}/cr_lower.yaml" + + pg_ver=$PG_VER + pg_exists=$(curl -s https://check.percona.com/versions/v1/pg-operator/${lower_ver} | jq '.versions[].matrix.postgresql | with_entries(select(.key | startswith("'$pg_ver'"))) | length') + if [ $pg_exists -eq 0 ]; then + pg_ver=$((pg_ver - 1)) + fi + echo $pg_ver >"${TEMP_DIR}/pg_ver.txt" + + backrest_image=$(get_version_images $lower_ver BACKREST${pg_ver}) + pgbouncer_image=$(get_version_images $lower_ver PGBOUNCER${pg_ver}) + postgres_image=$(get_version_images $lower_ver POSTGRESQL${pg_ver}) + operator_image=$(get_version_images $lower_ver OPERATOR) + + # pin the database-init image: without it InitImage() derives the image + # from the RUNNING operator's repo + the crVersion tag, so an operator + # under test from a non-release org would roll the instances on upgrade + yq -i eval ' + .metadata.name = "'${test_name}'" | + .metadata.labels = {"e2e":"'${test_name}'"} | + .spec.postgresVersion = '$pg_ver' | + .spec.initContainer.image = "'${REGISTRY_NAME_FULL}''${operator_image}'" | + .spec.users = [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | + .spec.image = "'${REGISTRY_NAME_FULL}''${postgres_image}'" | + .spec.proxy.pgBouncer.image = "'${REGISTRY_NAME_FULL}''${pgbouncer_image}'" | + .spec.backups.pgbackrest.image = "'${REGISTRY_NAME_FULL}''${backrest_image}'" | + .spec.backups.pgbackrest.configuration = [{"secret":{"name":"'${test_name}'-pgbackrest-secrets"}}] | + .spec.backups.pgbackrest.global."repo2-s3-uri-style" = "path" | + .spec.backups.pgbackrest.global."repo2-s3-verify-tls" = "n" | + .spec.backups.pgbackrest.repos += [{"name":"repo2","s3":{"bucket":"operator-testing","endpoint":"","region":"us-east-1"}}]' \ + "${TEMP_DIR}/cr_lower.yaml" + + MINIO_ENDPOINT="minio-service.${NAMESPACE}.svc.cluster.local:9000" + $sed -i "s||${MINIO_ENDPOINT}|g" "${TEMP_DIR}/cr_lower.yaml" + $sed -i 's|repo2-s3-verify-tls: n$|repo2-s3-verify-tls: "n"|' "${TEMP_DIR}/cr_lower.yaml" + + kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr_lower.yaml" diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/02-write-data-and-save-tls.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/02-write-data-and-save-tls.yaml new file mode 100644 index 0000000000..0d94546171 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/02-write-data-and-save-tls.yaml @@ -0,0 +1,39 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + run_psql_local 'CREATE DATABASE myapp' "$uri" || true + run_psql_local '\c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY)' "$uri" + run_psql_local '\c myapp \\\ INSERT INTO myApp VALUES (100500) ON CONFLICT DO NOTHING' "$uri" + + snapshot_tls_fingerprints ${test_name} ${TEMP_DIR}/tls.before + snapshot_workload_revisions ${test_name} ${TEMP_DIR}/revisions.before + + # sanity: PKI actually issued something + [[ -s ${TEMP_DIR}/tls.before ]] || { echo "ERROR: no TLS secrets found for baseline"; exit 1; } + + # premise check: the cluster MUST be operator-PKI issued at this point. + # The operator keeps PKI for an existing cluster only when its root CA secret + # carries no cert-manager annotation (see isRootCACertManagerManaged). + ca_annotation=$(kubectl -n "${NAMESPACE}" get secret ${test_name}-pgbackrest-ca \ + -o jsonpath='{.metadata.annotations.cert-manager\.io/certificate-name}' 2>/dev/null || true) + if [[ -z "$ca_annotation" ]]; then + ca_annotation=$(kubectl -n "${NAMESPACE}" get secrets -l postgres-operator.crunchydata.com/cluster=${test_name} \ + -o jsonpath='{range .items[*]}{.metadata.annotations.cert-manager\.io/certificate-name}{"\n"}{end}' | grep -v '^$' || true) + fi + if [[ -n "$ca_annotation" ]]; then + echo "ERROR: cluster TLS is already cert-manager managed ('$ca_annotation') - the PKI premise is broken." + echo "HINT: cert-manager was present when the cluster was created (leftover from a previous run?)." + exit 1 + fi + if [[ -n "$(kubectl -n "${NAMESPACE}" get certificates.cert-manager.io -o name 2>/dev/null)" ]]; then + echo "ERROR: cert-manager Certificates already exist before step 03 - the PKI premise is broken." + exit 1 + fi diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/03-deploy-cert-manager.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/03-deploy-cert-manager.yaml new file mode 100644 index 0000000000..5f592b3615 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/03-deploy-cert-manager.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # cert-manager appears in the cluster AFTER the PKI cluster exists — K8SPG-1017 trigger + deploy_cert_manager diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/04-assert.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/04-assert.yaml new file mode 100644 index 0000000000..f6f5223f83 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/04-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 480 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup-before-update +status: + state: Succeeded diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/04-backup.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/04-backup.yaml new file mode 100644 index 0000000000..e6c296da39 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/04-backup.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl -n "${NAMESPACE}" apply -f - << BACKUP + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + metadata: + name: backup-before-update + spec: + pgCluster: operator-upgrade-cert-manager + repoName: repo2 + options: + - --type=full + BACKUP diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/05-upgrade-operator.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/05-upgrade-operator.yaml new file mode 100644 index 0000000000..43b2acdf7a --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/05-upgrade-operator.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 240 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + date -u +%Y-%m-%dT%H:%M:%SZ > ${TEMP_DIR}/update_ts.txt + + update_operator + kubectl wait -n ${OPERATOR_NS:-$NAMESPACE} --timeout 60s \ + --for=jsonpath='{.spec.template.spec.containers[0].image}'=$IMAGE deployment/percona-postgresql-operator + + sleep 90 # several reconcile cycles under the new operator with cert-manager present diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/06-check-tls-unchanged.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/06-check-tls-unchanged.yaml new file mode 100644 index 0000000000..c7b6b3900f --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/06-check-tls-unchanged.yaml @@ -0,0 +1,32 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 240 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + wait_cluster_consistency ${test_name} + + # K8SPG-1017: no cert-manager Certificates may appear for a PKI-issued cluster + certs=$(kubectl -n "${NAMESPACE}" get certificates.cert-manager.io -o name 2>/dev/null || true) + if [[ -n "$certs" ]]; then + echo "ERROR: cert-manager Certificates appeared for the PKI cluster:" + echo "$certs" + exit 1 + fi + + # the operator must not hand the existing PKI root CA over to cert-manager + cm_annotated=$(kubectl -n "${NAMESPACE}" get secrets -l postgres-operator.crunchydata.com/cluster=${test_name} \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.annotations.cert-manager\.io/certificate-name}{"\n"}{end}' \ + | grep -v '=$' || true) + if [[ -n "$cm_annotated" ]]; then + echo "ERROR: TLS secrets became cert-manager managed after the upgrade:" + echo "$cm_annotated" + exit 1 + fi + + assert_tls_unchanged ${test_name} ${TEMP_DIR}/tls.before + assert_no_operator_rollout ${test_name} ${TEMP_DIR}/revisions.before diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/07-assert.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/07-assert.yaml new file mode 100644 index 0000000000..69bccec043 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/07-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 480 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup-after-update +status: + state: Succeeded diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/07-backup-after-update.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/07-backup-after-update.yaml new file mode 100644 index 0000000000..e07d536563 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/07-backup-after-update.yaml @@ -0,0 +1,19 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o xtrace + + kubectl -n "${NAMESPACE}" apply -f - << BACKUP + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + metadata: + name: backup-after-update + spec: + pgCluster: operator-upgrade-cert-manager + repoName: repo2 + options: + - --type=full + BACKUP diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/08-assert.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/08-assert.yaml new file mode 100644 index 0000000000..5cf047f3f3 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/08-assert.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore +metadata: + name: restore-after-update +status: + state: Succeeded +--- +# the cluster must be back to ready after the restore — step 09 connects +# to the primary right away, so restore completion alone is not enough +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: operator-upgrade-cert-manager +status: + state: ready + postgres: + ready: 3 + size: 3 + pgbouncer: + ready: 3 + size: 3 diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/08-restore.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/08-restore.yaml new file mode 100644 index 0000000000..3a4cbd2737 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/08-restore.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + # written AFTER backup-after-update — must disappear on restore + run_psql_local '\c myapp \\\ INSERT INTO myApp VALUES (100501) ON CONFLICT DO NOTHING' "$uri" + + # restore to the END OF THE BACKUP (--type=immediate), not to latest: + # a default restore replays all archived WAL and the marker would survive + primary=$(get_pod_by_role ${test_name} primary name) + latest_full_repo2_backup=$(kubectl -n ${NAMESPACE} exec ${primary} -- \ + pgbackrest info --output json --log-level-console=info \ + | jq '[.[] | .backup[] | select(.type == "full") | select(.database.["repo-key"] == 2)][-1].label') + + kubectl -n "${NAMESPACE}" apply -f - << RESTORE + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGRestore + metadata: + name: restore-after-update + spec: + pgCluster: ${test_name} + repoName: repo2 + options: + - --set=${latest_full_repo2_backup} + - --type=immediate + RESTORE diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/09-check-restored-data.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/09-check-restored-data.yaml new file mode 100644 index 0000000000..16788289d2 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/09-check-restored-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + data=$(run_psql_local '\c myapp \\\ SELECT id from myApp ORDER BY id' "$uri" | tr -d '[:space:]') + if [[ "$data" != "100500" ]]; then + echo "ERROR: unexpected data after restore: '$data' (want only 100500)" + exit 1 + fi diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/10-check-logs.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/10-check-logs.yaml new file mode 100644 index 0000000000..003b289acc --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/10-check-logs.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 90 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + check_logs_for_errors ${test_name} "$(cat ${TEMP_DIR}/update_ts.txt)" ../../tests/operator-upgrade-cert-manager/conf/log-allowlist.txt diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/operator-upgrade-cert-manager/99-remove-cluster-gracefully.yaml new file mode 100644 index 0000000000..511a63ed78 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/99-remove-cluster-gracefully.yaml @@ -0,0 +1,23 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster +- apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for c in operator-upgrade-cert-manager; do + kubectl -n ${NAMESPACE} patch postgrescluster $c --type=merge -p='{"metadata":{"finalizers":[]}}' 2>/dev/null || true + kubectl -n ${NAMESPACE} delete postgrescluster $c --ignore-not-found + done + remove_all_finalizers + check_operator_panic + destroy_operator + destroy_cert_manager + timeout: 120 diff --git a/e2e-tests/tests/operator-upgrade-cert-manager/conf/log-allowlist.txt b/e2e-tests/tests/operator-upgrade-cert-manager/conf/log-allowlist.txt new file mode 100644 index 0000000000..0ec51e5108 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-cert-manager/conf/log-allowlist.txt @@ -0,0 +1,17 @@ +the object has been modified; please apply your changes to the latest version +leader election lost +failed to sync secret cache +failed to sync configmap cache +Readiness probe +Liveness probe +FailedScheduling +ExternalProvisioning +WaitForFirstConsumer +WaitForPodScheduled +Multi-Attach error +TaintManagerEviction +UnableToCreateStanzas +unable to acquire lock +failed to cleanup outdated backups +get latest commit timestamp +the ResourceVersion in the precondition diff --git a/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml b/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml new file mode 100644 index 0000000000..098d743e4c --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-and-minio +timeout: 180 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 + - script: kubectl assert exist-enhanced deployment minio-service -n ${NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml b/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml new file mode 100644 index 0000000000..440d5143c6 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/00-deploy-operator.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + destroy_cert_manager + + git_tag="v$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" + deploy_operator_gh $git_tag + deploy_client + + # extension artifacts storage (same flow as the custom-extensions suite) + deploy_s3_secrets + deploy_minio + copy_custom_extensions_form_aws diff --git a/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml b/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml new file mode 100644 index 0000000000..afaa68357c --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/01-assert.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: operator-upgrade-extensions +status: + state: ready + postgres: + ready: 3 + size: 3 diff --git a/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml b/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml new file mode 100644 index 0000000000..d527e70e41 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/01-create-cluster.yaml @@ -0,0 +1,50 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + lower_ver="$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" + curl -s "https://raw.githubusercontent.com/percona/percona-postgresql-operator/v${lower_ver}/deploy/cr.yaml" >"${TEMP_DIR}/cr_lower.yaml" + + pg_ver=$PG_VER + pg_exists=$(curl -s https://check.percona.com/versions/v1/pg-operator/${lower_ver} | jq '.versions[].matrix.postgresql | with_entries(select(.key | startswith("'$pg_ver'"))) | length') + if [ $pg_exists -eq 0 ]; then + pg_ver=$((pg_ver - 1)) + fi + echo $pg_ver >"${TEMP_DIR}/pg_ver.txt" + + backrest_image=$(get_version_images $lower_ver BACKREST${pg_ver}) + pgbouncer_image=$(get_version_images $lower_ver PGBOUNCER${pg_ver}) + postgres_image=$(get_version_images $lower_ver POSTGRESQL${pg_ver}) + operator_image=$(get_version_images $lower_ver OPERATOR) + + yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' \ + "${TEMP_DIR}/cr_lower.yaml" ../../tests/operator-upgrade-extensions/conf/operator-upgrade-extensions.yaml \ + >"${TEMP_DIR}/cr.yaml" + + yq -i eval ' + .metadata.name = "'${test_name}'" | + .metadata.labels = {"e2e":"'${test_name}'"} | + .spec.postgresVersion = '$pg_ver' | + .spec.image = "'${REGISTRY_NAME_FULL}''${postgres_image}'" | + .spec.proxy.pgBouncer.image = "'${REGISTRY_NAME_FULL}''${pgbouncer_image}'" | + .spec.backups.pgbackrest.image = "'${REGISTRY_NAME_FULL}''${backrest_image}'"' "${TEMP_DIR}/cr.yaml" + + # extensions installer runs from the operator image of the RELEASED version + $sed -i "s||${REGISTRY_NAME_FULL}${operator_image}|g" "${TEMP_DIR}/cr.yaml" + + kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr.yaml" + + # released-CRD canary: unknown fields are silently pruned — fail loudly instead + applied_builtin=$(kubectl -n "${NAMESPACE}" get perconapgcluster ${test_name} -o jsonpath='{.spec.extensions.builtin}') + for ext in pg_stat_monitor pg_audit pgvector pg_repack; do + if ! echo "$applied_builtin" | grep -q "\"$ext\":true"; then + echo "ERROR: builtin extension '$ext' was pruned by the released CRD: $applied_builtin" + exit 1 + fi + done diff --git a/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml b/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml new file mode 100644 index 0000000000..84981087cd --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/02-add-custom-pg-cron.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 420 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kubectl -n "${NAMESPACE}" patch perconapgcluster ${test_name} --type merge -p '{ + "spec": { + "extensions": {"custom": [{"name": "pg_cron", "version": "1.6.6"}]}, + "patroni": {"dynamicConfiguration": {"postgresql": {"parameters": {"shared_preload_libraries": "pg_cron"}}}} + } + }' + sleep 7 # wait for reconcile loop + wait_cluster_consistency ${test_name} diff --git a/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml b/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml new file mode 100644 index 0000000000..597ec25187 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/03-enable-extensions-and-write-data.yaml @@ -0,0 +1,25 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + + for ext in pg_stat_monitor pgaudit vector pg_repack pg_cron; do + run_psql_local "\c postgres \\\\\\ CREATE EXTENSION IF NOT EXISTS \"$ext\"" "$uri" + done + + # workloads that must survive the operator update + run_psql_local 'CREATE TABLE IF NOT EXISTS cron_log (ts timestamptz)' "$uri" + run_psql_local 'SELECT cron.schedule($$tick$$,$$* * * * *$$,$$INSERT INTO cron_log VALUES (now())$$)' "$uri" + run_psql_local 'CREATE TABLE IF NOT EXISTS vec (e vector(3))' "$uri" + run_psql_local 'INSERT INTO vec VALUES ($$[1,2,3]$$)' "$uri" + + run_psql_local 'SELECT extname||$$=$$||extversion FROM pg_extension ORDER BY 1' "$uri" \ + | grep -v '^$' > ${TEMP_DIR}/ext.before + cat ${TEMP_DIR}/ext.before diff --git a/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml b/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml new file mode 100644 index 0000000000..b00e4ff7d4 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/04-upgrade-operator.yaml @@ -0,0 +1,18 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 240 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + snapshot_workload_revisions ${test_name} ${TEMP_DIR}/revisions.before + date -u +%Y-%m-%dT%H:%M:%SZ > ${TEMP_DIR}/update_ts.txt + + update_operator + kubectl wait -n ${OPERATOR_NS:-$NAMESPACE} --timeout 60s \ + --for=jsonpath='{.spec.template.spec.containers[0].image}'=$IMAGE deployment/percona-postgresql-operator + + sleep 90 # reconcile cycles + one cron minute boundary diff --git a/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml b/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml new file mode 100644 index 0000000000..7162121aad --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/05-check-extensions-survived.yaml @@ -0,0 +1,35 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + wait_cluster_consistency ${test_name} + assert_no_operator_rollout ${test_name} ${TEMP_DIR}/revisions.before + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + + run_psql_local 'SELECT extname||$$=$$||extversion FROM pg_extension ORDER BY 1' "$uri" \ + | grep -v '^$' > ${TEMP_DIR}/ext.after + diff -u ${TEMP_DIR}/ext.before ${TEMP_DIR}/ext.after || { echo "ERROR: pg_extension changed after operator update"; exit 1; } + + # cron is still ticking: wait for a new row past the update timestamp. + # poll instead of a fixed sleep — pg_cron fires on minute boundaries and + # the primary may still be settling right after the operator swap + fresh=0 + for i in $(seq 1 12); do + fresh=$(run_psql_local "SELECT count(*) FROM cron_log WHERE ts > \$\$$(cat ${TEMP_DIR}/update_ts.txt)\$\$" "$uri" | tr -d '[:space:]') + [[ "$fresh" -ge 1 ]] && break + sleep 15 + done + if [[ "$fresh" -lt 1 ]]; then + echo "ERROR: pg_cron stopped ticking after operator update" + exit 1 + fi + + vec=$(run_psql_local 'SELECT count(*) FROM vec' "$uri" | tr -d '[:space:]') + [[ "$vec" == "1" ]] || { echo "ERROR: vector data lost"; exit 1; } diff --git a/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml b/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml new file mode 100644 index 0000000000..b4f17d2112 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/06-upgrade-cluster.yaml @@ -0,0 +1,39 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 420 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + pg_ver_lower="$(cat ${TEMP_DIR}/pg_ver.txt)" + target_image_postgresql=$IMAGE_POSTGRESQL + target_image_pgbouncer=$IMAGE_PGBOUNCER + target_image_backrest=$IMAGE_BACKREST + + if [ "$pg_ver_lower" != "$PG_VER" ]; then + target_image_postgresql="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-ppg$pg_ver_lower-postgres" + target_image_pgbouncer="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-pgbouncer$pg_ver_lower" + target_image_backrest="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-pgbackrest$pg_ver_lower" + fi + + cr_ver=$(yq '.spec.crVersion' $DEPLOY_DIR/cr.yaml) + + kubectl -n "${NAMESPACE}" patch perconapgcluster ${test_name} --type merge -p '{ + "spec": { + "crVersion": "'${cr_ver}'", + "image": "'${target_image_postgresql}'", + "proxy": {"pgBouncer": {"image": "'${target_image_pgbouncer}'"}}, + "backups": {"pgbackrest": {"image": "'${target_image_backrest}'"}}, + "extensions": {"image": "'${IMAGE}'"}, + "initContainer": {"image": "'${IMAGE}'"} + } + }' + sleep 7 # wait for reconcile loop + + for s in $(kubectl get sts -n $NAMESPACE --no-headers -l postgres-operator.crunchydata.com/instance-set=instance1 --output=custom-columns='NAME:.metadata.name'); do + kubectl wait -n $NAMESPACE --timeout 300s --for=jsonpath='{.spec.template.spec.containers[?(@.name=="database")].image}'=$target_image_postgresql sts/$s + done + wait_cluster_consistency ${test_name} diff --git a/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml b/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml new file mode 100644 index 0000000000..bf7d6d7add --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/07-check-extensions-after-cluster-upgrade.yaml @@ -0,0 +1,35 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + + run_psql_local 'SELECT extname||$$=$$||extversion FROM pg_extension ORDER BY 1' "$uri" \ + | grep -v '^$' > ${TEMP_DIR}/ext.upgraded + # the cluster upgrade brings newer extension builds and the operator legitimately + # runs ALTER EXTENSION ... UPDATE — so compare the extension SET only, + # versions are allowed to move forward (no process substitution: kuttl + # runs steps with sh, which doesn't support it) + cut -d= -f1 ${TEMP_DIR}/ext.before > ${TEMP_DIR}/extnames.before + cut -d= -f1 ${TEMP_DIR}/ext.upgraded > ${TEMP_DIR}/extnames.upgraded + diff -u ${TEMP_DIR}/extnames.before ${TEMP_DIR}/extnames.upgraded \ + || { echo "ERROR: extension set changed after cluster upgrade"; exit 1; } + + # poll instead of a fixed sleep — pg_cron fires on minute boundaries and + # the primary may still be settling right after the cluster upgrade rollout + fresh=0 + for i in $(seq 1 12); do + fresh=$(run_psql_local 'SELECT count(*) FROM cron_log WHERE ts > now() - interval $$2 minutes$$' "$uri" | tr -d '[:space:]') + [[ "$fresh" -ge 1 ]] && break + sleep 15 + done + [[ "$fresh" -ge 1 ]] || { echo "ERROR: pg_cron stopped ticking after cluster upgrade"; exit 1; } + + vec=$(run_psql_local 'SELECT count(*) FROM vec' "$uri" | tr -d '[:space:]') + [[ "$vec" == "1" ]] || { echo "ERROR: vector data lost after cluster upgrade"; exit 1; } diff --git a/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml b/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml new file mode 100644 index 0000000000..80f67980ee --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/08-check-logs.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 90 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + check_logs_for_errors ${test_name} "$(cat ${TEMP_DIR}/update_ts.txt)" ../../tests/operator-upgrade-extensions/conf/log-allowlist.txt diff --git a/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml new file mode 100644 index 0000000000..9b24d3ca80 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/99-remove-cluster-gracefully.yaml @@ -0,0 +1,20 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster +- apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kubectl -n ${NAMESPACE} patch postgrescluster operator-upgrade-extensions --type=merge -p='{"metadata":{"finalizers":[]}}' 2>/dev/null || true + kubectl -n ${NAMESPACE} delete postgrescluster operator-upgrade-extensions --ignore-not-found + remove_all_finalizers + check_operator_panic + destroy_operator + timeout: 60 diff --git a/e2e-tests/tests/operator-upgrade-extensions/conf/log-allowlist.txt b/e2e-tests/tests/operator-upgrade-extensions/conf/log-allowlist.txt new file mode 100644 index 0000000000..0ec51e5108 --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/conf/log-allowlist.txt @@ -0,0 +1,17 @@ +the object has been modified; please apply your changes to the latest version +leader election lost +failed to sync secret cache +failed to sync configmap cache +Readiness probe +Liveness probe +FailedScheduling +ExternalProvisioning +WaitForFirstConsumer +WaitForPodScheduled +Multi-Attach error +TaintManagerEviction +UnableToCreateStanzas +unable to acquire lock +failed to cleanup outdated backups +get latest commit timestamp +the ResourceVersion in the precondition diff --git a/e2e-tests/tests/operator-upgrade-extensions/conf/operator-upgrade-extensions.yaml b/e2e-tests/tests/operator-upgrade-extensions/conf/operator-upgrade-extensions.yaml new file mode 100644 index 0000000000..77e657cdfe --- /dev/null +++ b/e2e-tests/tests/operator-upgrade-extensions/conf/operator-upgrade-extensions.yaml @@ -0,0 +1,29 @@ +spec: + initContainer: + image: + patroni: + dynamicConfiguration: + postgresql: + parameters: + cron.use_background_workers: "on" + users: + - name: postgres + password: + type: AlphaNumeric + extensions: + image: + imagePullPolicy: Always + storage: + type: s3 + bucket: operator-testing + region: us-east-1 + endpoint: http://minio-service:9000 + forcePathStyle: true + disableSSL: true + secret: + name: minio-secret + builtin: + pg_stat_monitor: true + pg_audit: true + pgvector: true + pg_repack: true diff --git a/e2e-tests/tests/upgrade-minor/00-deploy-operator.yaml b/e2e-tests/tests/upgrade-minor/00-deploy-operator.yaml index f940fa77d9..165243dc55 100644 --- a/e2e-tests/tests/upgrade-minor/00-deploy-operator.yaml +++ b/e2e-tests/tests/upgrade-minor/00-deploy-operator.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 10 +timeout: 300 commands: - script: |- set -o errexit @@ -8,7 +8,36 @@ commands: source ../../functions init_temp_dir # do this only in the first TestStep + destroy_cert_manager git_tag="v$(curl -s https://check.percona.com/versions/v1/pg-operator | jq -r '.versions[].operator' | sort -V | tail -n1)" deploy_operator_gh $git_tag deploy_client + + # minio + pgbackrest credentials for repo2 (cross-version backup/restore) + kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/minio-secret.yml" + access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret \ + -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)" + secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret \ + -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)" + + openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ + -keyout "${TEMP_DIR}/minio.key" \ + -out "${TEMP_DIR}/minio.crt" \ + -subj "/CN=minio-service" + kubectl -n "${NAMESPACE}" create secret generic minio-tls \ + --from-file=public.crt="${TEMP_DIR}/minio.crt" \ + --from-file=private.key="${TEMP_DIR}/minio.key" + + deploy_minio minio-service minio-tls + + cat > "${TEMP_DIR}/pgbackrest-minio.ini" << INI + [global] + repo2-s3-key=${access_key} + repo2-s3-key-secret=${secret_key} + INI + kubectl -n "${NAMESPACE}" create secret generic upgrade-minor-pgbackrest-secrets \ + --from-file=cloud.conf="${TEMP_DIR}/pgbackrest-minio.ini" + + kubectl -n "${NAMESPACE}" create configmap upgrade-minor-init-sql \ + --from-literal=init.sql='CREATE SCHEMA IF NOT EXISTS app;' diff --git a/e2e-tests/tests/upgrade-minor/01-assert.yaml b/e2e-tests/tests/upgrade-minor/01-assert.yaml index 763e98b46c..31974580d5 100644 --- a/e2e-tests/tests/upgrade-minor/01-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/01-assert.yaml @@ -1,6 +1,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 120 +# the max-settings cluster (3 replicas, wal/tablespace volumes, repo2) plus the +# replica-create backup needs noticeably longer than the old single-repo setup +timeout: 600 --- kind: StatefulSet apiVersion: apps/v1 @@ -129,6 +131,8 @@ status: name: repo1 replicaCreateBackupComplete: true stanzaCreated: true + - name: repo2 + stanzaCreated: true proxy: pgBouncer: readyReplicas: 3 diff --git a/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml b/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml index 1dcf1a4e51..53538bb1dd 100644 --- a/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml +++ b/e2e-tests/tests/upgrade-minor/01-create-cluster.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 100 +timeout: 120 commands: - script: |- set -o errexit @@ -31,15 +31,26 @@ commands: backrest_image=$(get_version_images $lower_ver BACKREST${pg_ver}) pgbouncer_image=$(get_version_images $lower_ver PGBOUNCER${pg_ver}) postgres_image=$(get_version_images $lower_ver POSTGRESQL${pg_ver}) + operator_image=$(get_version_images $lower_ver OPERATOR) + + yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' \ + "${TEMP_DIR}/cr_lower.yaml" ../../tests/upgrade-minor/conf/upgrade-minor.yaml \ + >"${TEMP_DIR}/cr.yaml" yq -i eval ' .metadata.name = "'${test_name}'" | .metadata.labels = {"e2e":"'${test_name}'"} | - .spec.image = "'${REGISTRY_NAME_FULL}''${postgres_image}'" | .spec.postgresVersion = '$pg_ver' | + .spec.initContainer.image = "'${REGISTRY_NAME_FULL}''${operator_image}'" | + .spec.image = "'${REGISTRY_NAME_FULL}''${postgres_image}'" | .spec.proxy.pgBouncer.image = "'${REGISTRY_NAME_FULL}''${pgbouncer_image}'" | - .spec.backups.pgbackrest.repos[0].schedules = null | - .spec.backups.pgbackrest.image = "'${REGISTRY_NAME_FULL}''${backrest_image}'" | - .spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | - .spec.users += [{"name":"'${test_name}'","password":{"type":"AlphaNumeric"}}]' "${TEMP_DIR}/cr_lower.yaml" - kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr_lower.yaml" + .spec.backups.pgbackrest.image = "'${REGISTRY_NAME_FULL}''${backrest_image}'"' "${TEMP_DIR}/cr.yaml" + + MINIO_ENDPOINT="minio-service.${NAMESPACE}.svc.cluster.local:9000" + $sed -i "s||${MINIO_ENDPOINT}|g" "${TEMP_DIR}/cr.yaml" + # yq may unquote "n" turning it into YAML boolean — restore the quoting + $sed -i 's|repo2-s3-verify-tls: n$|repo2-s3-verify-tls: "n"|' "${TEMP_DIR}/cr.yaml" + + yq '.spec.crVersion' "${TEMP_DIR}/cr_lower.yaml" >"${TEMP_DIR}/cr_ver_released.txt" + + kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr.yaml" diff --git a/e2e-tests/tests/upgrade-minor/02-assert.yaml b/e2e-tests/tests/upgrade-minor/02-assert.yaml new file mode 100644 index 0000000000..910a2bcff6 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/02-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup-before-update +status: + state: Succeeded diff --git a/e2e-tests/tests/upgrade-minor/02-write-data.yaml b/e2e-tests/tests/upgrade-minor/02-write-data.yaml index f9ba9a7079..c9a4d3f3b6 100644 --- a/e2e-tests/tests/upgrade-minor/02-write-data.yaml +++ b/e2e-tests/tests/upgrade-minor/02-write-data.yaml @@ -1,5 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep +timeout: 120 commands: - script: |- set -o errexit @@ -10,7 +11,21 @@ commands: run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:$(get_psql_user_pass upgrade-minor-pguser-postgres)@$(get_psql_user_host upgrade-minor-pguser-postgres)" - + run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass upgrade-minor-pguser-postgres)@$(get_psql_user_host upgrade-minor-pguser-postgres)" + + # full backup taken by the RELEASED operator (restored later in step 08 + # by the operator under test — cross-version backup compatibility) + kubectl -n "${NAMESPACE}" apply -f - << BACKUP + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + metadata: + name: backup-before-update + spec: + pgCluster: ${test_name} + repoName: repo2 + options: + - --type=full + BACKUP diff --git a/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml b/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml index 7d1a67b0e1..8e17ee1379 100644 --- a/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml +++ b/e2e-tests/tests/upgrade-minor/04-upgrade-operator.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep -timeout: 30 +timeout: 180 commands: - script: |- set -o errexit @@ -8,5 +8,11 @@ commands: source ../../functions + # reference state for the no-rollout check in step 05 + snapshot_workload_revisions ${test_name} ${TEMP_DIR}/revisions.before + date -u +%Y-%m-%dT%H:%M:%SZ > ${TEMP_DIR}/update_ts.txt + # background writes: step 05 asserts the cluster stayed available + start_heartbeat ${test_name} + update_operator kubectl wait -n ${OPERATOR_NS:-$NAMESPACE} --timeout 30s --for=jsonpath='{.spec.template.spec.containers[0].image}'=$IMAGE deployment/percona-postgresql-operator diff --git a/e2e-tests/tests/upgrade-minor/05-assert.yaml b/e2e-tests/tests/upgrade-minor/05-assert.yaml index a3bc8603ee..48e13e5aec 100644 --- a/e2e-tests/tests/upgrade-minor/05-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/05-assert.yaml @@ -85,7 +85,14 @@ status: apiVersion: upstream.pgv2.percona.com/v1beta1 kind: PostgresCluster metadata: - generation: 1 + # 3 = create + manual-backup options written by the operator (step 02) + # + the first spec write by the upgraded operator. + # That last write is triggered by the custom-extensions-synced annotation + # (K8SPG-786) and would not change the spec by itself — but the pg_tde + # field serializes as "pg_tde: {}" even when unused (omitempty is a no-op + # on value structs), so the write lands a spec diff and bumps generation. + # Once pg_tde becomes a pointer this goes back to 2. + generation: 3 labels: e2e: upgrade-minor name: upgrade-minor @@ -103,7 +110,7 @@ status: readyReplicas: 3 replicas: 3 updatedReplicas: 3 - observedGeneration: 1 + observedGeneration: 3 pgbackrest: repoHost: apiVersion: apps/v1 @@ -114,6 +121,8 @@ status: name: repo1 replicaCreateBackupComplete: true stanzaCreated: true + - name: repo2 + stanzaCreated: true proxy: pgBouncer: readyReplicas: 3 @@ -123,7 +132,7 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: name: upgrade-minor - generation: 1 + generation: 2 status: conditions: - type: ReadyForBackup diff --git a/e2e-tests/tests/upgrade-minor/05-check-no-rollout.yaml b/e2e-tests/tests/upgrade-minor/05-check-no-rollout.yaml new file mode 100644 index 0000000000..6ad9bacec4 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/05-check-no-rollout.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + wait_cluster_consistency ${test_name} + # operator upgrade alone must not restart a single workload pod + assert_no_operator_rollout ${test_name} ${TEMP_DIR}/revisions.before + # writes kept flowing the whole time (also stops the heartbeat) + assert_heartbeat_gap ${test_name} 30 diff --git a/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml b/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml deleted file mode 100644 index 03415f6015..0000000000 --- a/e2e-tests/tests/upgrade-minor/05-sleep-after-operator-update.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - timeout: 105 - script: |- - sleep 90 diff --git a/e2e-tests/tests/upgrade-minor/06-check-logs.yaml b/e2e-tests/tests/upgrade-minor/06-check-logs.yaml new file mode 100644 index 0000000000..b37706567e --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/06-check-logs.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # no errors "in between": operator + postgres logs since the update + check_logs_for_errors ${test_name} "$(cat ${TEMP_DIR}/update_ts.txt)" ../../tests/upgrade-minor/conf/log-allowlist.txt diff --git a/e2e-tests/tests/upgrade-minor/07-assert.yaml b/e2e-tests/tests/upgrade-minor/07-assert.yaml new file mode 100644 index 0000000000..fc1c511a7f --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/07-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGBackup +metadata: + name: backup-old-crversion +status: + state: Succeeded diff --git a/e2e-tests/tests/upgrade-minor/07-backup-old-crversion.yaml b/e2e-tests/tests/upgrade-minor/07-backup-old-crversion.yaml new file mode 100644 index 0000000000..7586074047 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/07-backup-old-crversion.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # backups must work while the NEW operator manages a cluster that still + # carries the RELEASED crVersion — that is the point of this step + current_cr_ver=$(kubectl -n "${NAMESPACE}" get perconapgcluster ${test_name} -o jsonpath='{.spec.crVersion}') + released_cr_ver=$(cat ${TEMP_DIR}/cr_ver_released.txt) + if [[ "$current_cr_ver" != "$released_cr_ver" ]]; then + echo "ERROR: crVersion changed unexpectedly: $current_cr_ver != $released_cr_ver" + exit 1 + fi + + kubectl -n "${NAMESPACE}" apply -f - << BACKUP + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGBackup + metadata: + name: backup-old-crversion + spec: + pgCluster: ${test_name} + repoName: repo2 + options: + - --type=full + BACKUP diff --git a/e2e-tests/tests/upgrade-minor/08-assert.yaml b/e2e-tests/tests/upgrade-minor/08-assert.yaml index 2e9b11ee57..285b55996e 100644 --- a/e2e-tests/tests/upgrade-minor/08-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/08-assert.yaml @@ -1,12 +1,20 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 600 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 07-read-from-primary -data: - data: |2- - 100500 - 100501 + name: restore-after-update +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: upgrade-minor +status: + state: ready + postgres: + ready: 3 + size: 3 diff --git a/e2e-tests/tests/upgrade-minor/08-restore.yaml b/e2e-tests/tests/upgrade-minor/08-restore.yaml new file mode 100644 index 0000000000..c8a565dd4c --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/08-restore.yaml @@ -0,0 +1,34 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + + # marker written AFTER the last backup — must disappear on restore + run_psql_local '\c myapp \\\ INSERT INTO myApp VALUES (100501) ON CONFLICT DO NOTHING' "$uri" + + # restore to the END OF THE BACKUP (--type=immediate), not to latest: + # a default restore replays all archived WAL and the marker would survive + primary=$(get_pod_by_role ${test_name} primary name) + latest_full_repo2_backup=$(kubectl -n ${NAMESPACE} exec ${primary} -- \ + pgbackrest info --output json --log-level-console=info \ + | jq '[.[] | .backup[] | select(.type == "full") | select(.database.["repo-key"] == 2)][-1].label') + + kubectl -n "${NAMESPACE}" apply -f - << RESTORE + apiVersion: pgv2.percona.com/v2 + kind: PerconaPGRestore + metadata: + name: restore-after-update + spec: + pgCluster: ${test_name} + repoName: repo2 + options: + - --set=${latest_full_repo2_backup} + - --type=immediate + RESTORE diff --git a/e2e-tests/tests/upgrade-minor/06-assert.yaml b/e2e-tests/tests/upgrade-minor/09-assert.yaml similarity index 86% rename from e2e-tests/tests/upgrade-minor/06-assert.yaml rename to e2e-tests/tests/upgrade-minor/09-assert.yaml index f009790bc7..83e0b4ce22 100644 --- a/e2e-tests/tests/upgrade-minor/06-assert.yaml +++ b/e2e-tests/tests/upgrade-minor/09-assert.yaml @@ -32,7 +32,7 @@ status: kind: StatefulSet apiVersion: apps/v1 metadata: - generation: 2 + generation: 3 labels: app.kubernetes.io/component: pg app.kubernetes.io/instance: upgrade-minor @@ -64,7 +64,7 @@ spec: status: availableReplicas: 1 collisionCount: 0 - observedGeneration: 2 + observedGeneration: 3 readyReplicas: 1 replicas: 1 updatedReplicas: 1 @@ -112,7 +112,12 @@ status: apiVersion: upstream.pgv2.percona.com/v1beta1 kind: PostgresCluster metadata: - generation: 2 + # 7 = create + first spec write by the upgraded operator (step 04, see the + # comment in 05-assert.yaml: "pg_tde: {}" rides along and bumps the + # generation) + 2 manual-backup options writes (steps 02, 07) + # + restore spec write and cleanup (step 08) + cluster upgrade (step 09). + # Once pg_tde becomes a pointer this goes back to 6. + generation: 7 labels: e2e: upgrade-minor name: upgrade-minor @@ -130,7 +135,7 @@ status: readyReplicas: 3 replicas: 3 updatedReplicas: 3 - observedGeneration: 2 + observedGeneration: 7 pgbackrest: repoHost: apiVersion: apps/v1 @@ -141,6 +146,8 @@ status: name: repo1 replicaCreateBackupComplete: true stanzaCreated: true + - name: repo2 + stanzaCreated: true proxy: pgBouncer: readyReplicas: 3 @@ -149,8 +156,11 @@ status: apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: - generation: 2 name: upgrade-minor + # 6 = create + 2 manual-backup options writes (steps 02, 07) + # + restore patch (step 08) + the operator resetting restore.enabled + # after it finishes + cluster upgrade apply (step 09) + generation: 6 status: pgbouncer: ready: 3 diff --git a/e2e-tests/tests/upgrade-minor/06-upgrade-cluster.yaml b/e2e-tests/tests/upgrade-minor/09-upgrade-cluster.yaml similarity index 63% rename from e2e-tests/tests/upgrade-minor/06-upgrade-cluster.yaml rename to e2e-tests/tests/upgrade-minor/09-upgrade-cluster.yaml index 49cccd600e..7837d768af 100644 --- a/e2e-tests/tests/upgrade-minor/06-upgrade-cluster.yaml +++ b/e2e-tests/tests/upgrade-minor/09-upgrade-cluster.yaml @@ -1,5 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep +timeout: 420 commands: - script: |- set -o errexit @@ -7,6 +8,13 @@ commands: source ../../functions + uri="postgres:$(get_psql_user_pass ${test_name}-pguser-postgres)@$(get_psql_user_host ${test_name}-pguser-postgres)" + data=$(run_psql_local '\c myapp \\\ SELECT id from myApp ORDER BY id' "$uri" | tr -d '[:space:]') + if [[ "$data" != "100500" ]]; then + echo "ERROR: unexpected data after restore: '$data' (want only 100500)" + exit 1 + fi + pg_ver_lower="$(cat ${TEMP_DIR}/pg_ver.txt)" target_image_postgresql=$IMAGE_POSTGRESQL target_image_pgbouncer=$IMAGE_PGBOUNCER @@ -14,25 +22,23 @@ commands: if [ "$pg_ver_lower" != "$PG_VER" ]; then target_image_postgresql="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-ppg$pg_ver_lower-postgres" - target_image_pgbouncer="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-ppg$pg_ver_lower-pgbouncer" - target_image_backrest="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-ppg$pg_ver_lower-pgbackrest" + target_image_pgbouncer="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-pgbouncer$pg_ver_lower" + target_image_backrest="${REGISTRY_NAME_FULL}perconalab/percona-postgresql-operator:main-pgbackrest$pg_ver_lower" fi cr_ver=$(yq '.spec.crVersion' $DEPLOY_DIR/cr.yaml) yq -i eval ' - .metadata.name = "'${test_name}'" | - .metadata.labels = {"e2e":"'${test_name}'"} | .spec.crVersion = "'${cr_ver}'" | + .spec.initContainer.image = "'${IMAGE}'" | .spec.image = "'${target_image_postgresql}'" | .spec.proxy.pgBouncer.image = "'${target_image_pgbouncer}'" | - .spec.backups.pgbackrest.image = "'${target_image_backrest}'"' "${TEMP_DIR}/cr_lower.yaml" - kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr_lower.yaml" + .spec.backups.pgbackrest.image = "'${target_image_backrest}'"' "${TEMP_DIR}/cr.yaml" + kubectl -n "${NAMESPACE}" apply -f "${TEMP_DIR}/cr.yaml" sleep 7 # wait for reconcile loop kubectl wait -n $NAMESPACE --timeout 180s --for=jsonpath='{.spec.template.spec.containers[0].image}'=$target_image_backrest sts/upgrade-minor-repo-host kubectl wait -n $NAMESPACE --timeout 180s --for=jsonpath='{.spec.template.spec.containers[0].image}'=$target_image_pgbouncer deployment/upgrade-minor-pgbouncer for s in $(kubectl get sts -n $NAMESPACE --no-headers -l postgres-operator.crunchydata.com/instance-set=instance1 --output=custom-columns='NAME:.metadata.name'); do - kubectl wait -n $NAMESPACE --timeout 180s --for=jsonpath='{.spec.template.spec.containers[0].image}'=$target_image_postgresql sts/${s} + kubectl wait -n $NAMESPACE --timeout 300s --for=jsonpath='{.spec.template.spec.containers[?(@.name=="database")].image}'=$target_image_postgresql sts/${s} done - timeout: 180 - + wait_cluster_consistency ${test_name} diff --git a/e2e-tests/tests/upgrade-minor/07-write-data.yaml b/e2e-tests/tests/upgrade-minor/10-write-data.yaml similarity index 100% rename from e2e-tests/tests/upgrade-minor/07-write-data.yaml rename to e2e-tests/tests/upgrade-minor/10-write-data.yaml diff --git a/e2e-tests/tests/upgrade-minor/11-assert.yaml b/e2e-tests/tests/upgrade-minor/11-assert.yaml new file mode 100644 index 0000000000..5a8d353c9c --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/11-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 30 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: 11-read-from-primary +data: + data: |2- + 100500 + 100501 diff --git a/e2e-tests/tests/upgrade-minor/08-read-from-primary.yaml b/e2e-tests/tests/upgrade-minor/11-read-from-primary.yaml similarity index 85% rename from e2e-tests/tests/upgrade-minor/08-read-from-primary.yaml rename to e2e-tests/tests/upgrade-minor/11-read-from-primary.yaml index 17b6c261f4..9f4e903588 100644 --- a/e2e-tests/tests/upgrade-minor/08-read-from-primary.yaml +++ b/e2e-tests/tests/upgrade-minor/11-read-from-primary.yaml @@ -10,4 +10,4 @@ commands: data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass upgrade-minor-pguser-postgres)@$(get_psql_user_host upgrade-minor-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 07-read-from-primary --from-literal=data="${data}" + kubectl create configmap -n "${NAMESPACE}" 11-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/upgrade-minor/09-check-collation-mismatch.yaml b/e2e-tests/tests/upgrade-minor/12-check-collation-mismatch.yaml similarity index 92% rename from e2e-tests/tests/upgrade-minor/09-check-collation-mismatch.yaml rename to e2e-tests/tests/upgrade-minor/12-check-collation-mismatch.yaml index b7c1803974..70a6ee8aa4 100644 --- a/e2e-tests/tests/upgrade-minor/09-check-collation-mismatch.yaml +++ b/e2e-tests/tests/upgrade-minor/12-check-collation-mismatch.yaml @@ -14,7 +14,7 @@ commands: fi # Replace new lines, whitespaces and quotes - query_mismatch=$(<09-collation-mismatch.sql tr '\n' ' ' | $sed "s/'/'\\\\''/g" | $sed 's/ */ /g') + query_mismatch=$(<12-collation-mismatch.sql tr '\n' ' ' | $sed "s/'/'\\\\''/g" | $sed 's/ */ /g') data=$(run_psql_local "$query_mismatch" "postgres:$(get_psql_user_pass upgrade-minor-pguser-postgres)@$(get_psql_user_host upgrade-minor-pguser-postgres)") if [[ "$data" == *"Collection mismatch detected"* ]]; then diff --git a/e2e-tests/tests/upgrade-minor/09-collation-mismatch.sql b/e2e-tests/tests/upgrade-minor/12-collation-mismatch.sql similarity index 100% rename from e2e-tests/tests/upgrade-minor/09-collation-mismatch.sql rename to e2e-tests/tests/upgrade-minor/12-collation-mismatch.sql diff --git a/e2e-tests/tests/upgrade-minor/13-check-logs.yaml b/e2e-tests/tests/upgrade-minor/13-check-logs.yaml new file mode 100644 index 0000000000..a7cd83ba5e --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/13-check-logs.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 120 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + # no errors "after": the window now also covers the cluster upgrade + check_logs_for_errors ${test_name} "$(cat ${TEMP_DIR}/update_ts.txt)" ../../tests/upgrade-minor/conf/log-allowlist.txt diff --git a/e2e-tests/tests/upgrade-minor/conf/log-allowlist.txt b/e2e-tests/tests/upgrade-minor/conf/log-allowlist.txt new file mode 100644 index 0000000000..0ec51e5108 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/conf/log-allowlist.txt @@ -0,0 +1,17 @@ +the object has been modified; please apply your changes to the latest version +leader election lost +failed to sync secret cache +failed to sync configmap cache +Readiness probe +Liveness probe +FailedScheduling +ExternalProvisioning +WaitForFirstConsumer +WaitForPodScheduled +Multi-Attach error +TaintManagerEviction +UnableToCreateStanzas +unable to acquire lock +failed to cleanup outdated backups +get latest commit timestamp +the ResourceVersion in the precondition diff --git a/e2e-tests/tests/upgrade-minor/conf/upgrade-minor.yaml b/e2e-tests/tests/upgrade-minor/conf/upgrade-minor.yaml new file mode 100644 index 0000000000..80392ec907 --- /dev/null +++ b/e2e-tests/tests/upgrade-minor/conf/upgrade-minor.yaml @@ -0,0 +1,102 @@ +spec: + instances: + - name: instance1 + replicas: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + walVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + tablespaceVolumes: + - name: user + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + sidecars: + - name: test-sidecar + image: busybox:latest + command: ["sleep", "30d"] + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + users: + - name: postgres + password: + type: AlphaNumeric + - name: upgrade-minor + password: + type: AlphaNumeric + - name: extra + databases: + - extradb + password: + type: ASCII + patroni: + dynamicConfiguration: + postgresql: + parameters: + shared_buffers: 128MB + max_connections: "200" + proxy: + pgBouncer: + replicas: 3 + sidecars: + - name: test-sidecar + image: busybox:latest + command: ["sleep", "30d"] + config: + global: + pool_mode: session + databaseInitSQL: + key: init.sql + name: upgrade-minor-init-sql + backups: + pgbackrest: + configuration: + - secret: + name: upgrade-minor-pgbackrest-secrets + global: + repo2-retention-full: "3" + repo2-s3-uri-style: path + repo2-s3-verify-tls: "n" + manual: + repoName: repo2 + options: + - --type=full + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: repo2 + s3: + bucket: operator-testing + endpoint: + region: us-east-1 diff --git a/internal/controller/postgrescluster/postgres.go b/internal/controller/postgrescluster/postgres.go index 885b26b437..807c3610fc 100644 --- a/internal/controller/postgrescluster/postgres.go +++ b/internal/controller/postgrescluster/postgres.go @@ -12,6 +12,7 @@ import ( "net" "net/url" "regexp" + "slices" "sort" "strconv" "strings" @@ -353,11 +354,17 @@ func (r *Reconciler) reconcilePostgresDatabases( } } + // dropping needs the builtin flag and the custom extensions list to + // agree: while the extension is in spec.extensions.custom the custom + // machinery owns it and it must be left alone (same for set_user below) + extensionsSynced := cluster.Annotations[v1beta1.AnnotationCustomExtensionsSynced] == "true" if cluster.Spec.Extensions.PGCron { if pgCronOK = pgcron.EnableInPostgreSQL(ctx, exec) == nil; !pgCronOK { r.Recorder.Event(cluster, corev1.EventTypeWarning, "pgCronDisabled", "Unable to install pg_cron") } + } else if !extensionsSynced || slices.Contains(cluster.Spec.Extensions.Custom, "pg_cron") { + pgCronOK = true } else { if pgCronOK = pgcron.DisableInPostgreSQL(ctx, exec) == nil; !pgCronOK { r.Recorder.Event(cluster, corev1.EventTypeWarning, "pgCronEnabled", @@ -370,6 +377,8 @@ func (r *Reconciler) reconcilePostgresDatabases( r.Recorder.Event(cluster, corev1.EventTypeWarning, "setUserDisabled", "Unable to install set_user") } + } else if !extensionsSynced || slices.Contains(cluster.Spec.Extensions.Custom, "set_user") { + setUserOK = true } else { if setUserOK = setuser.DisableInPostgreSQL(ctx, exec) == nil; !setUserOK { r.Recorder.Event(cluster, corev1.EventTypeWarning, "setUserEnabled", diff --git a/percona/controller/pgcluster/controller.go b/percona/controller/pgcluster/controller.go index 36653d7b68..9d1324e84a 100644 --- a/percona/controller/pgcluster/controller.go +++ b/percona/controller/pgcluster/controller.go @@ -848,6 +848,19 @@ func (r *PGClusterReconciler) handleMonitorUserPassChange(ctx context.Context, c return nil } +func builtInExtensionEnabled(cr *v2.PerconaPGCluster, name string) bool { + extensions := cr.Spec.Extensions + + switch name { + case "pg_cron": + return ptr.Deref(extensions.PGCron.Enabled, false) + case "set_user": + return ptr.Deref(extensions.SetUser.Enabled, false) + } + + return false +} + func (r *PGClusterReconciler) ensureMonitorUserSecret(ctx context.Context, cr *v2.PerconaPGCluster) (*corev1.Secret, error) { log := logging.FromContext(ctx) @@ -931,9 +944,17 @@ func (r *PGClusterReconciler) reconcileCustomExtensions(ctx context.Context, cr // Check for missing entries in crExtensions for _, ext := range installedExtensions { // If an object exists in installedExtensions but not in crExtensions, the extension should be deleted. - if _, ok := crExtensions[ext]; !ok { - removedExtensions = append(removedExtensions, ext) + if _, ok := crExtensions[ext]; ok { + continue + } + // ...unless the user moved it from spec.extensions.custom to + // spec.extensions.builtin. Dropping it here would destroy the data + // the extension owns (e.g. the cron.job rows) and the builtin + // reconcile would then re-create it empty. + if builtInExtensionEnabled(cr, ext) { + continue } + removedExtensions = append(removedExtensions, ext) } if len(removedExtensions) > 0 { diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index e20feb493c..97763f7fab 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -603,6 +603,8 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun annotations[naming.AutoCreateUserSchemaAnnotation] = "true" } + annotations[crunchyv1beta1.AnnotationCustomExtensionsSynced] = "true" + postgresCluster.Annotations = annotations postgresCluster.Labels = cr.Labels if postgresCluster.Labels == nil { @@ -728,6 +730,11 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun postgresCluster.Spec.Extensions.SetUser = *cr.Spec.Extensions.SetUser.Enabled } + postgresCluster.Spec.Extensions.Custom = nil + for _, ext := range cr.Spec.Extensions.Custom { + postgresCluster.Spec.Extensions.Custom = append(postgresCluster.Spec.Extensions.Custom, ext.Name) + } + postgresCluster.Spec.TLSOnly = cr.Spec.TLSOnly postgresCluster.Spec.TLS = cr.Spec.TLS diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go index f35a64121c..13fb72dd76 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go @@ -571,6 +571,168 @@ func TestPerconaPGCluster_ToCrunchy(t *testing.T) { assert.Nil(t, actual.Spec.Proxy) }, }, + "custom extension is not dropped by builtin reconcile": { + expectedPerconaPGCluster: &PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: PerconaPGClusterSpec{ + CRVersion: version.Version(), + PostgresVersion: 18, + Extensions: ExtensionsSpec{ + Custom: []CustomExtensionSpec{ + {Name: "pg_cron", Version: "1.6.6"}, + }, + }, + InstanceSets: PGInstanceSets{ + { + Name: "instance1", + Replicas: &[]int32{1}[0], + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + }, + Backups: Backups{ + PGBackRest: PGBackRestArchive{ + Repos: []crunchyv1beta1.PGBackRestRepo{}, + }, + }, + }, + }, + assertClusterFunc: func(t *testing.T, actual *crunchyv1beta1.PostgresCluster, _ *PerconaPGCluster) { + // an unset flag means false like every other builtin flag, but + // the custom names travel along: pg_cron is owned by the custom + // machinery, so the reconcile must not drop it despite the flag + assert.False(t, actual.Spec.Extensions.PGCron) + assert.Equal(t, []string{"pg_cron"}, actual.Spec.Extensions.Custom) + }, + }, + "explicit builtin flags are passed through": { + expectedPerconaPGCluster: &PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: PerconaPGClusterSpec{ + CRVersion: version.Version(), + PostgresVersion: 18, + Extensions: ExtensionsSpec{ + PGCron: BuiltInExtensionSpec{Enabled: new(true)}, + SetUser: BuiltInExtensionSpec{Enabled: new(false)}, + // pg_cron is listed here as well: an explicit builtin + // flag wins, the user asked the operator to manage it + Custom: []CustomExtensionSpec{ + {Name: "pg_cron", Version: "1.6.6"}, + }, + }, + InstanceSets: PGInstanceSets{ + { + Name: "instance1", + Replicas: &[]int32{1}[0], + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + }, + Backups: Backups{ + PGBackRest: PGBackRestArchive{ + Repos: []crunchyv1beta1.PGBackRestRepo{}, + }, + }, + }, + }, + assertClusterFunc: func(t *testing.T, actual *crunchyv1beta1.PostgresCluster, _ *PerconaPGCluster) { + assert.True(t, actual.Spec.Extensions.PGCron) + // an explicit false must still drop the extension, otherwise + // there would be no way to uninstall a builtin one + assert.False(t, actual.Spec.Extensions.SetUser) + }, + }, + "explicit false does not drop an extension still listed as custom": { + expectedPerconaPGCluster: &PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: PerconaPGClusterSpec{ + CRVersion: version.Version(), + PostgresVersion: 18, + Extensions: ExtensionsSpec{ + PGCron: BuiltInExtensionSpec{Enabled: new(false)}, + Custom: []CustomExtensionSpec{ + {Name: "pg_cron", Version: "1.6.6"}, + }, + }, + InstanceSets: PGInstanceSets{ + { + Name: "instance1", + Replicas: &[]int32{1}[0], + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + }, + Backups: Backups{ + PGBackRest: PGBackRestArchive{ + Repos: []crunchyv1beta1.PGBackRestRepo{}, + }, + }, + }, + }, + assertClusterFunc: func(t *testing.T, actual *crunchyv1beta1.PostgresCluster, _ *PerconaPGCluster) { + // dropping requires the builtin flag and the custom list to + // agree: the flag is false, but the custom entry still owns the + // extension, so the reconcile must leave it alone + assert.False(t, actual.Spec.Extensions.PGCron) + assert.Equal(t, []string{"pg_cron"}, actual.Spec.Extensions.Custom) + }, + }, + "unset builtin flags fall back to false": { + expectedPerconaPGCluster: &PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: PerconaPGClusterSpec{ + CRVersion: version.Version(), + PostgresVersion: 18, + InstanceSets: PGInstanceSets{ + { + Name: "instance1", + Replicas: &[]int32{1}[0], + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }, + }, + Backups: Backups{ + PGBackRest: PGBackRestArchive{ + Repos: []crunchyv1beta1.PGBackRestRepo{}, + }, + }, + }, + }, + inputPostgresCluster: &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: crunchyv1beta1.PostgresClusterSpec{ + Extensions: crunchyv1beta1.ExtensionsSpec{ + PGCron: true, + SetUser: true, + Custom: []string{"pg_cron"}, + }, + }, + }, + assertClusterFunc: func(t *testing.T, actual *crunchyv1beta1.PostgresCluster, _ *PerconaPGCluster) { + assert.False(t, actual.Spec.Extensions.PGCron) + assert.False(t, actual.Spec.Extensions.SetUser) + assert.Empty(t, actual.Spec.Extensions.Custom) + }, + }, } for testName, tt := range tests { diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go index 15c38db34a..7e597ac124 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go @@ -308,6 +308,8 @@ type ExtensionsSpec struct { PGCron bool `json:"pgCron,omitempty"` SetUser bool `json:"setUser,omitempty"` + Custom []string `json:"custom,omitempty"` + PGTDE PGTDESpec `json:"pg_tde,omitempty"` } @@ -1066,6 +1068,8 @@ func NewPostgresCluster() *PostgresCluster { const LabelVersion = "pgv2.percona.com/version" +const AnnotationCustomExtensionsSynced = "pgv2.percona.com/custom-extensions-synced" + func (cr *PostgresCluster) CompareVersion(ver string) int { crVersion, err := gover.NewVersion(cr.Labels[LabelVersion]) if err != nil { diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go index 3e3d364913..dd791b95a1 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go @@ -436,6 +436,11 @@ func (in *ExporterSpec) DeepCopy() *ExporterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExtensionsSpec) DeepCopyInto(out *ExtensionsSpec) { *out = *in + if in.Custom != nil { + in, out := &in.Custom, &out.Custom + *out = make([]string, len(*in)) + copy(*out, *in) + } in.PGTDE.DeepCopyInto(&out.PGTDE) }