From e5bedcd828c85b0d464e74c4dd9223d3b151dfb9 Mon Sep 17 00:00:00 2001 From: Utkarsh Srivastava Date: Fri, 26 Jul 2024 14:40:28 +0530 Subject: [PATCH] Added a workaround for postgres upgrade failures (#1396) (#1398) If Postgres is not upgraded to postgres-15 and is still using postgres-12 image, continue to use postgres12 image, but set the noobaa CR as rejected. This is to prevent noobaa-db going into CLBO Signed-off-by: Danny Zaken (cherry picked from commit 992d39bfafd516138dc3fe70a12f40e51f4a4805) Co-authored-by: Danny Zaken --- Makefile | 2 +- build/gen-odf-package.sh | 5 ++++- pkg/olm/olm.go | 4 ++++ pkg/options/options.go | 8 ++++++++ pkg/system/phase2_creating.go | 2 +- pkg/system/reconciler.go | 22 ++++++++++++++++------ pkg/system/system.go | 19 ++++++++++++++++++- 7 files changed, 52 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index b8ea7da17..06235a1a3 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ gen-olm: gen gen-odf-package: cli rm -rf $(MANIFESTS) - MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" build/gen-odf-package.sh + MANIFESTS="$(MANIFESTS)" CSV_NAME="$(csv-name)" SKIP_RANGE="$(skip-range)" REPLACES="$(replaces)" CORE_IMAGE="$(core-image)" DB_IMAGE="$(db-image)" OPERATOR_IMAGE="$(operator-image)" COSI_SIDECAR_IMAGE="$(cosi-sidecar-image)" OBC_CRD="$(obc-crd)" PSQL_12_IMAGE="$(psql-12-image)" build/gen-odf-package.sh @echo "✅ gen-odf-package" .PHONY: gen-odf-package diff --git a/build/gen-odf-package.sh b/build/gen-odf-package.sh index 0705d01cc..d2ff44e9f 100755 --- a/build/gen-odf-package.sh +++ b/build/gen-odf-package.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ] +if [ "${MANIFESTS}" == "" ] || [ "${CSV_NAME}" == "" ] || [ "${CORE_IMAGE}" == "" ] || [ "${PSQL_12_IMAGE}" == "" ] || [ "${DB_IMAGE}" == "" ] || [ "${OPERATOR_IMAGE}" == "" ] || [ "${COSI_SIDECAR_IMAGE}" == "" ] then echo "gen-odf-package.sh: not all required envs were supplied" exit 1 @@ -16,6 +16,7 @@ echo "--obc-crd=${OBC_CRD}" --replaces "${REPLACES}" \ --noobaa-image ${CORE_IMAGE} \ --db-image ${DB_IMAGE} \ +--psql-12-image ${PSQL_12_IMAGE} \ --operator-image ${OPERATOR_IMAGE} \ --cosi-sidecar-image ${COSI_SIDECAR_IMAGE} \ --obc-crd=${OBC_CRD} @@ -34,6 +35,8 @@ cat >> ${temp_csv} << EOF name: noobaa-core - image: ${DB_IMAGE} name: noobaa-db + - image: ${PSQL_12_IMAGE} + name: noobaa-psql-12 - image: ${OPERATOR_IMAGE} name: noobaa-operator EOF diff --git a/pkg/olm/olm.go b/pkg/olm/olm.go index 3de461227..593ab8a96 100644 --- a/pkg/olm/olm.go +++ b/pkg/olm/olm.go @@ -297,6 +297,10 @@ func GenerateCSV(opConf *operator.Conf, csvParams *generateCSVParams) *operv1.Cl Name: "NOOBAA_DB_IMAGE", Value: options.DBImage, }, + corev1.EnvVar{ + Name: "NOOBAA_PSQL_12_IMAGE", + Value: options.Psql12Image, + }, corev1.EnvVar{ Name: "ENABLE_NOOBAA_ADMISSION", Value: "true", diff --git a/pkg/options/options.go b/pkg/options/options.go index 217af1435..ebb9e46b1 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -74,6 +74,10 @@ var NooBaaImage = ContainerImage // it can be overridden for testing or different registry locations. var DBImage = "quay.io/sclorg/postgresql-15-c9s" +// Psql12Image is the default postgres12 db image url +// currently it can not be overridden. +var Psql12Image = "centos/postgresql-12-centos7" + // DBType is the default db image type // it can be overridden for testing or different types. var DBType = "postgres" @@ -214,6 +218,10 @@ func init() { &DBImage, "db-image", DBImage, "The database container image", ) + FlagSet.StringVar( + &Psql12Image, "psql-12-image", + Psql12Image, "The database old container image", + ) FlagSet.StringVar( &CosiSideCarImage, "cosi-sidecar-image", CosiSideCarImage, "The cosi side car container image", diff --git a/pkg/system/phase2_creating.go b/pkg/system/phase2_creating.go index 691cb1845..bdea4a8dc 100644 --- a/pkg/system/phase2_creating.go +++ b/pkg/system/phase2_creating.go @@ -247,7 +247,7 @@ func (r *Reconciler) SetDesiredNooBaaDB() error { for i := range podSpec.Containers { c := &podSpec.Containers[i] if c.Name == "db" { - c.Image = GetDesiredDBImage(r.NooBaa) + c.Image = GetDesiredDBImage(r.NooBaa, c.Image) if r.NooBaa.Spec.DBResources != nil { c.Resources = *r.NooBaa.Spec.DBResources } diff --git a/pkg/system/reconciler.go b/pkg/system/reconciler.go index f060098ff..24cd5b9b1 100644 --- a/pkg/system/reconciler.go +++ b/pkg/system/reconciler.go @@ -421,12 +421,22 @@ func (r *Reconciler) Reconcile() (reconcile.Result, error) { log.Warnf("⏳ Temporary Error: %s", err) } } else { - r.SetPhase( - nbv1.SystemPhaseReady, - "SystemPhaseReady", - "noobaa operator completed reconcile - system is ready", - ) - log.Infof("✅ Done") + // Postgres upgrade failure workaround + // if the system reconciliation has no other error, but the postgres image is still postresql-12 image, set the status to Rejected + if IsPostgresql12Image(r.NooBaaPostgresDB.Spec.Template.Spec.Containers[0].Image) { + r.SetPhase(nbv1.SystemPhaseRejected, + "PostgresImageVersion", + "Noobaa is using Postgresql-12 which indicates a failure to upgrade to Postgresql-15. Please contact support.") + log.Errorf("❌ Postgres image version is set to postgresql-12. Indicates a failure to upgrade to Postgresql-15. Please contact support.") + } else { + r.SetPhase( + nbv1.SystemPhaseReady, + "SystemPhaseReady", + "noobaa operator completed reconcile - system is ready", + ) + log.Infof("✅ Done") + } + } err = r.UpdateStatus() diff --git a/pkg/system/system.go b/pkg/system/system.go index 11ebcc2d6..83785e2aa 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -1116,7 +1116,19 @@ func Connect(isExternal bool) (*Client, error) { } // GetDesiredDBImage returns the desired DB image according to spec or env or default (in options) -func GetDesiredDBImage(sys *nbv1.NooBaa) string { +func GetDesiredDBImage(sys *nbv1.NooBaa, currentImage string) string { + // Postgres upgrade failure workaround + // if the current Postgres image is a postgresql-12 image, use NOOBAA_PSQL12_IMAGE. otherwise use GetdesiredDBImage + if IsPostgresql12Image(currentImage) { + psql12Image, ok := os.LookupEnv("NOOBAA_PSQL_12_IMAGE") + util.Logger().Warnf("The current Postgres image is a postgresql-12 image. using (%s)", psql12Image) + if !ok { + psql12Image = currentImage + util.Logger().Warnf("NOOBAA_PSQL_12_IMAGE is not set. using the current image %s", currentImage) + } + return psql12Image + } + if sys.Spec.DBImage != nil { return *sys.Spec.DBImage } @@ -1128,6 +1140,11 @@ func GetDesiredDBImage(sys *nbv1.NooBaa) string { return options.DBImage } +// IsPostgresql12Image checks if the image is a postgresql-12 image +func IsPostgresql12Image(image string) bool { + return strings.Contains(image, "postgresql-12") +} + // CheckSystem checks the state of the system and initializes its status fields func CheckSystem(sys *nbv1.NooBaa) bool { log := util.Logger()