K8SPG-786 K8SPG-1040 add upgrade test and fix operator behaviuor with extentions.

config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml

@@ -6193,6 +6193,10 @@ spec:
                 type: boolean
               extensions:
                 properties:
+                  custom:
+                    items:
+                      type: string
+                    type: array
                   pg_tde:
                     properties:
                       enabled:

deploy/bundle.yaml

@@ -27103,6 +27103,10 @@ spec:
                 type: boolean
               extensions:
                 properties:
+                  custom:
+                    items:
+                      type: string
+                    type: array
                   pg_tde:
                     properties:
                       enabled:

deploy/crd.yaml

@@ -27103,6 +27103,10 @@ spec:
                 type: boolean
               extensions:
                 properties:
+                  custom:
+                    items:
+                      type: string
+                    type: array
                   pg_tde:
                     properties:
                       enabled:

deploy/cw-bundle.yaml

@@ -27103,6 +27103,10 @@ spec:
                 type: boolean
               extensions:
                 properties:
+                  custom:
+                    items:
+                      type: string
+                    type: array
                   pg_tde:
                     properties:
                       enabled:

e2e-tests/functions

@@ -347,7 +347,7 @@ function deploy_s3_secrets() {
 	kubectl -n "$NAMESPACE" create secret generic "${test_name}-pgbackrest-secrets" "${secret_args[@]}"
 
 	case $test_name in
-		"custom-extensions" | "builtin-extensions" | major-upgrade*)
+		"custom-extensions" | "builtin-extensions" | "operator-upgrade-extensions" | major-upgrade*)
 			kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/cloud-secret.yml"
 			kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/minio-secret.yml"
 			;;
@@ -2298,6 +2298,119 @@ function verify_k8s_nodes_version() {
 	done
 }
 
+# Save sts update revisions + deployment revisions of the cluster workloads.
+snapshot_workload_revisions() {
+	local cluster="$1" out="$2"
+	kubectl -n "$NAMESPACE" get sts -l postgres-operator.crunchydata.com/cluster="$cluster" \
+		-o jsonpath='{range .items[*]}{.metadata.name}={.status.updateRevision}{"\n"}{end}' >"$out"
+	kubectl -n "$NAMESPACE" get deploy -l postgres-operator.crunchydata.com/cluster="$cluster" \
+		-o jsonpath='{range .items[*]}{.metadata.name}={.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' >>"$out"
+}
+
+# Fail if any sts/deploy revision changed (operator-driven rollout).
+# Container restarts are tolerated only on pods whose node was evicted.
+assert_no_operator_rollout() {
+	local cluster="$1" baseline="$2" current="${TEMP_DIR}/revisions.current"
+	snapshot_workload_revisions "$cluster" "$current"
+	if ! diff -u "$baseline" "$current"; then
+		echo "ERROR: workload revisions changed - operator triggered a rollout"
+		return 1
+	fi
+	local restarted
+	restarted=$(kubectl -n "$NAMESPACE" get pods -l postgres-operator.crunchydata.com/cluster="$cluster" \
+		-o jsonpath='{range .items[*]}{.metadata.name}={.status.containerStatuses[*].restartCount}{"\n"}{end}' \
+		| grep -Ev '=(0 ?)*0?$' | cut -d= -f1 || true)
+	local pod
+	for pod in $restarted; do
+		if ! kubectl -n "$NAMESPACE" get events --field-selector involvedObject.name="$pod" \
+			-o jsonpath='{range .items[*]}{.reason}{"\n"}{end}' | grep -qE 'TaintManagerEviction|NodeNotReady|Preempt'; then
+			echo "ERROR: pod $pod restarted without node eviction"
+			return 1
+		fi
+	done
+	return 0
+}
+
+# sha256 of tls.crt of every TLS-bearing secret of the cluster.
+snapshot_tls_fingerprints() {
+	local cluster="$1" out="$2" s crt
+	: >"$out"
+	for s in $(kubectl -n "$NAMESPACE" get secrets -l postgres-operator.crunchydata.com/cluster="$cluster" -o name); do
+		crt=$(kubectl -n "$NAMESPACE" get "$s" -o jsonpath='{.data.tls\.crt}')
+		if [[ -n $crt ]]; then
+			echo "${s}=$(echo "$crt" | sha256sum | awk '{print $1}')" >>"$out"
+		fi
+	done
+	sort -o "$out" "$out"
+}
+
+assert_tls_unchanged() {
+	local current="${TEMP_DIR}/tls.current"
+	snapshot_tls_fingerprints "$1" "$current"
+	if ! diff -u "$2" "$current"; then
+		echo "ERROR: TLS certificates were re-issued"
+		return 1
+	fi
+}
+
+# Background INSERT loop through pgbouncer; pid stored in TEMP_DIR.
+start_heartbeat() {
+	local cluster="$1"
+	local uri="postgres:$(get_psql_user_pass ${cluster}-pguser-postgres)@$(get_psql_user_host ${cluster}-pguser-postgres)"
+	run_psql_local 'CREATE TABLE IF NOT EXISTS heartbeat (ts timestamptz)' "$uri"
+	(
+		set +o xtrace
+		while true; do
+			run_psql_local 'INSERT INTO heartbeat VALUES (now())' "$uri" || true
+			sleep 2
+		done
+	) >/dev/null 2>&1 &
+	echo $! >"${TEMP_DIR}/heartbeat.pid"
+}
+
+assert_heartbeat_gap() {
+	local cluster="$1" max_gap="$2"
+	kill "$(cat "${TEMP_DIR}/heartbeat.pid")" 2>/dev/null || true
+	local uri="postgres:$(get_psql_user_pass ${cluster}-pguser-postgres)@$(get_psql_user_host ${cluster}-pguser-postgres)"
+	local gap
+	gap=$(run_psql_local 'SELECT COALESCE(ceil(EXTRACT(EPOCH FROM max(ts - prev)))::int,0) FROM (SELECT ts, lag(ts) OVER (ORDER BY ts) AS prev FROM heartbeat) g' "$uri" | tr -d '[:space:]')
+	echo "heartbeat max gap: ${gap}s (limit ${max_gap}s)"
+	if [[ -z "$gap" || "$gap" -gt "$max_gap" ]]; then
+		echo "ERROR: heartbeat gap '${gap}' exceeds ${max_gap}s"
+		return 1
+	fi
+}
+
+# Scan operator + postgres logs and Warning events since a timestamp.
+check_logs_for_errors() {
+	local cluster="$1" since="$2" allow="$3" fail=0 pod f
+	local op_ns="${OPERATOR_NS:-$NAMESPACE}"
+	rm -f "${TEMP_DIR}"/log_errors_*.txt
+	# the operator logs JSON only with LOG_STRUCTURED=true, deploy/operator.yaml
+	# sets it to false, so match the console encoder format as well
+	kubectl -n "$op_ns" logs deployment/percona-postgresql-operator --since-time="$since" 2>/dev/null \
+		| grep -E '"level":"error"|[[:space:]]ERROR[[:space:]]|panic:' | grep -Evf "$allow" >"${TEMP_DIR}/log_errors_operator.txt" || true
+	for pod in $(kubectl -n "$NAMESPACE" get pods -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres -o name); do
+		kubectl -n "$NAMESPACE" logs "$pod" -c database --since-time="$since" 2>/dev/null \
+			| grep -E 'FATAL|PANIC' | grep -Evf "$allow" >>"${TEMP_DIR}/log_errors_postgres.txt" || true
+	done
+	# event messages can be multi-line (pgBackRest output): fold every event back
+	# into a single line, otherwise the continuation lines pass both the
+	# timestamp filter and the allowlist, which only matches the first line
+	kubectl -n "$NAMESPACE" get events --field-selector type=Warning \
+		-o jsonpath='{range .items[*]}{.lastTimestamp} {.involvedObject.name} {.reason} {.message}{"\n"}{end}' 2>/dev/null \
+		| awk '/^[0-9]{4}-[0-9]{2}-[0-9]{2}T/ {if (rec != "") print rec; rec = $0; next} {rec = rec " " $0} END {if (rec != "") print rec}' \
+		| awk -v s="$since" '$1 >= s' | grep -Evf "$allow" >"${TEMP_DIR}/log_errors_events.txt" || true
+	for f in operator postgres events; do
+		if [[ -s "${TEMP_DIR}/log_errors_${f}.txt" ]]; then
+			echo "=== unexpected ${f} errors since ${since} ==="
+			cat "${TEMP_DIR}/log_errors_${f}.txt"
+			fail=1
+		fi
+	done
+	return $fail
+}
+
 function vault_tls() {
 	local name=${1:-vault-service}
 	local tmp_dir=$2

e2e-tests/run-pr.csv

@@ -22,6 +22,8 @@ ldap
 ldap-tls
 logcollection
 monitoring
+operator-upgrade-cert-manager
+operator-upgrade-extensions
 one-pod
 repo-host-autogrow
 operator-self-healing

e2e-tests/run-release.csv

@@ -27,6 +27,8 @@ ldap
 ldap-tls
 logcollection
 monitoring
+operator-upgrade-cert-manager
+operator-upgrade-extensions
 one-pod
 repo-host-autogrow
 operator-self-healing

e2e-tests/tests/custom-extensions/11-assert.yaml

@@ -1,6 +1,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 120
+timeout: 240
 ---
 kind: StatefulSet
 apiVersion: apps/v1

e2e-tests/tests/custom-extensions/17-assert.yaml

@@ -0,0 +1,25 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+# adding pg_cron to the custom list and to shared_preload_libraries restarts
+# every instance pod
+timeout: 420
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: custom-extensions
+status:
+  installedCustomExtensions:
+  - pg_stat_kcache
+  - pg_cron
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/custom-extensions/17-install-pg_cron.yaml

@@ -0,0 +1,20 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: custom-extensions
+spec:
+  # pg_cron is installed as a custom extension here so the next steps can move
+  # it to spec.extensions.pg_cron: it is the only extension that is both
+  # available as a custom archive and managed as a builtin one, and it owns
+  # data (the cron.job rows) that a wrong drop would destroy
+  extensions:
+    custom:
+    - name: pg_stat_kcache
+      version: 2.3.2
+    - name: pg_cron
+      version: 1.6.7
+  patroni:
+    dynamicConfiguration:
+      postgresql:
+        parameters:
+          shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron

e2e-tests/tests/custom-extensions/18-restart-cluster.yaml

@@ -0,0 +1,16 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      # pg_cron was added to shared_preload_libraries in the previous step and
+      # postgres only loads it on start, so the pods have to be restarted before
+      # the extension can be created
+      wait_cluster_consistency custom-extensions
+
+      restart_pg_pods custom-extensions instance1
+    timeout: 360

e2e-tests/tests/custom-extensions/19-enable-pg_cron.yaml

@@ -0,0 +1,31 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 120
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)"
+
+      # cron.database_name only exists once the pg_cron library is loaded, so it
+      # tells us the restart of the previous step actually picked up the new
+      # shared_preload_libraries. CREATE EXTENSION fails without it.
+      for i in $(seq 1 30); do
+          if run_psql_local '\c postgres \\\ SHOW cron.database_name' "$uri" >/dev/null 2>&1; then
+              break
+          fi
+          sleep 5
+      done
+      run_psql_local '\c postgres \\\ SHOW cron.database_name' "$uri" \
+          || { echo "ERROR: pg_cron library is not loaded, shared_preload_libraries did not take effect"; exit 1; }
+
+      run_psql_local '\c postgres \\\ CREATE EXTENSION pg_cron' "$uri"
+
+      # a job in cron.job is the data a user loses if pg_cron gets dropped
+      run_psql_local '\c postgres \\\ SELECT cron.schedule($$migration-tick$$,$$* * * * *$$,$$SELECT 1$$)' "$uri"
+
+      jobs=$(run_psql_local '\c postgres \\\ SELECT count(*) FROM cron.job WHERE jobname = $$migration-tick$$' "$uri" | tr -d '[:space:]')
+      [[ "$jobs" == "1" ]] || { echo "ERROR: failed to schedule the pg_cron job"; exit 1; }

e2e-tests/tests/custom-extensions/20-assert.yaml

@@ -0,0 +1,25 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+# removing pg_cron from the custom list changes INSTALL_EXTENSIONS and restarts
+# every instance pod
+timeout: 600
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: custom-extensions
+status:
+  # pg_cron is gone from the custom list, it is managed as a builtin now
+  installedCustomExtensions:
+  - pg_stat_kcache
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/custom-extensions/20-migrate-to-builtin.yaml

@@ -0,0 +1,20 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: custom-extensions
+spec:
+  # pg_cron moves from a custom extension to a builtin one: the operator takes
+  # over its lifecycle, but the extension and its data must survive the move.
+  # Dropping it here used to be the default behavior, because the extension
+  # disappears from the custom list.
+  extensions:
+    pg_cron:
+      enabled: true
+    custom:
+    - name: pg_stat_kcache
+      version: 2.3.2
+  patroni:
+    dynamicConfiguration:
+      postgresql:
+        parameters:
+          shared_preload_libraries: pg_stat_statements,pg_stat_kcache,pg_cron

e2e-tests/tests/custom-extensions/21-check-migration.yaml

@@ -0,0 +1,20 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 120
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      uri="postgres:$(get_psql_user_pass custom-extensions-pguser-postgres)@$(get_psql_user_host custom-extensions-pguser-postgres)"
+
+      # moving an extension from spec.extensions.custom to a builtin flag must
+      # not drop it: the operator removes it from the custom list, but the
+      # builtin flag is enabled, so the extension and its data have to stay
+      installed=$(run_psql_local '\c postgres \\\ SELECT extname FROM pg_extension WHERE extname = $$pg_cron$$' "$uri" | tr -d '[:space:]')
+      [[ "$installed" == "pg_cron" ]] || { echo "ERROR: pg_cron was dropped by the custom to builtin migration"; exit 1; }
+
+      jobs=$(run_psql_local '\c postgres \\\ SELECT count(*) FROM cron.job WHERE jobname = $$migration-tick$$' "$uri" | tr -d '[:space:]')
+      [[ "$jobs" == "1" ]] || { echo "ERROR: cron.job data lost in the custom to builtin migration"; exit 1; }

e2e-tests/tests/operator-upgrade-cert-manager/00-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-and-minio
+timeout: 180
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1
+  - script: kubectl assert exist-enhanced deployment minio-service -n ${NAMESPACE} --field-selector status.readyReplicas=1