diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 01a68127..235fd5e0 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -82,7 +82,7 @@ jobs: needs: unit-test strategy: matrix: - k8s_version: ['v1.19.12', 'v1.20.12', 'v1.21.6', 'v1.22.3'] + k8s_version: ['v1.19.12', 'v1.20.12', 'v1.21.6', 'v1.22.3', 'v1.23.1'] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/charts/db-operator/Chart.yaml b/charts/db-operator/Chart.yaml index 373fcee2..5cdcbca5 100644 --- a/charts/db-operator/Chart.yaml +++ b/charts/db-operator/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 type: application -kubeVersion: ">= 1.19-prerelease <= 1.22-prerelease" -appVersion: "1.4.0" +kubeVersion: ">= 1.19-prerelease <= 1.23-prerelease" +appVersion: "1.5.0" description: A Database Operator name: db-operator -version: 1.1.4 +version: 1.1.5 diff --git a/charts/db-operator/templates/operator.yaml b/charts/db-operator/templates/operator.yaml index 363ad5a4..0d8d8377 100644 --- a/charts/db-operator/templates/operator.yaml +++ b/charts/db-operator/templates/operator.yaml @@ -41,7 +41,7 @@ spec: - name: OPERATOR_NAME value: "db-operator" - name: WATCH_NAMESPACE - value: "" # it's necessary to set "" to watch cluster wide + value: {{ .Values.watchNamespace | quote }} - name: RECONCILE_INTERVAL value: {{ .Values.reconcileInterval | quote }} - name: POD_NAME diff --git a/charts/db-operator/values.yaml b/charts/db-operator/values.yaml index d9252a56..feaba20c 100644 --- a/charts/db-operator/values.yaml +++ b/charts/db-operator/values.yaml @@ -8,6 +8,8 @@ image: # - name: myRegistrySecret reconcileInterval: "60" +# watchNamespace value is comma-separated list of namespace names. It's necessary to set "" to watch cluster wide. +watchNamespace: "" rbac: create: true diff --git a/controllers/database_controller.go b/controllers/database_controller.go index ef290ec3..af925eeb 100644 --- a/controllers/database_controller.go +++ b/controllers/database_controller.go @@ -21,6 +21,9 @@ import ( "errors" "io/ioutil" "os" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" "strconv" "time" @@ -48,11 +51,12 @@ import ( // DatabaseReconciler reconciles a Database object type DatabaseReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - Recorder record.EventRecorder - Interval time.Duration - Conf *config.Config + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + Interval time.Duration + Conf *config.Config + WatchNamespaces []string } var ( @@ -150,7 +154,13 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return reconcileResult, nil } - if isDBSpecChanged(dbcr) { + databaseSecret, err := r.getDatabaseSecret(ctx, dbcr) + if err != nil && !k8serrors.IsNotFound(err) { + logrus.Errorf("could not get database secret - %s", err) + return r.manageError(ctx, dbcr, err, true) + } + + if isDBChanged(dbcr, databaseSecret) { logrus.Infof("DB: namespace=%s, name=%s spec changed", dbcr.Namespace, dbcr.Name) err := r.initialize(ctx, dbcr) if err != nil { @@ -162,7 +172,7 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return r.manageError(ctx, dbcr, err, true) } - addDBSpecChecksum(dbcr) + addDBChecksum(dbcr, databaseSecret) err = r.Update(ctx, dbcr) if err != nil { logrus.Errorf("error resource updating - %s", err) @@ -248,8 +258,24 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // SetupWithManager sets up the controller with the Manager. func (r *DatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error { + + eventFilter := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return isWatchedNamespace(r.WatchNamespaces, e.Object) && isDatabase(e.Object) + }, // Reconcile only Database Create Event + DeleteFunc: func(e event.DeleteEvent) bool { + return isWatchedNamespace(r.WatchNamespaces, e.Object) && isDatabase(e.Object) + }, // Reconcile only Database Delete Event + UpdateFunc: func(e event.UpdateEvent) bool { + return isWatchedNamespace(r.WatchNamespaces, e.ObjectNew) && isObjectUpdated(e) + }, // Reconcile Database and Secret Update Events + GenericFunc: func(e event.GenericEvent) bool { return true }, // Reconcile any Generic Events (operator POD or cluster restarted) + } + return ctrl.NewControllerManagedBy(mgr). For(&kciv1alpha1.Database{}). + WithEventFilter(eventFilter). + Watches(&source.Kind{Type: &corev1.Secret{}}, &secretEventHandler{r.Client}). Complete(r) } @@ -342,6 +368,12 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph return err } + err = r.annotateDatabaseSecret(ctx, dbcr, databaseSecret) + if err != nil { + logrus.Errorf("could not annotate database secret - %s", err) + return err + } + dbcr.Status.DatabaseName = databaseCred.Name dbcr.Status.UserName = databaseCred.Username logrus.Infof("DB: namespace=%s, name=%s successfully created", dbcr.Namespace, dbcr.Name) @@ -677,6 +709,17 @@ func (r *DatabaseReconciler) getDatabaseSecret(ctx context.Context, dbcr *kciv1a return secret, nil } +func (r *DatabaseReconciler) annotateDatabaseSecret(ctx context.Context, dbcr *kciv1alpha1.Database, secret *corev1.Secret) error { + annotations := secret.ObjectMeta.GetAnnotations() + if len(annotations) == 0 { + annotations = make(map[string]string) + } + annotations[DbSecretAnnotation] = dbcr.Name + secret.ObjectMeta.SetAnnotations(annotations) + + return r.Update(ctx, secret) +} + func (r *DatabaseReconciler) getAdminSecret(ctx context.Context, dbcr *kciv1alpha1.Database) (*corev1.Secret, error) { instance, err := dbcr.GetInstanceRef() if err != nil { diff --git a/controllers/database_secret_handler.go b/controllers/database_secret_handler.go new file mode 100644 index 00000000..a6f43b01 --- /dev/null +++ b/controllers/database_secret_handler.go @@ -0,0 +1,148 @@ +/* + * Copyright 2021 kloeckner.i GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controllers + +import ( + kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "strings" +) + +const ( + DbSecretAnnotation = "db-operator/database" +) + +/* ------ Secret Event Handler ------ */ +type secretEventHandler struct { + client.Client +} + +func (e *secretEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + + logrus.Info("Start processing Database Secret Update Event") + + switch v := evt.ObjectNew.(type) { + + default: + logrus.Error("Database Secret Update Event error! Unknown object: type=", v.GetObjectKind(), ", name=", evt.ObjectNew.GetNamespace(), "/", evt.ObjectNew.GetName()) + return + + case *corev1.Secret: + // only annotated secrets are watched + secretNew := evt.ObjectNew.(*corev1.Secret) + annotations := secretNew.ObjectMeta.GetAnnotations() + dbSecretAnnotation, ok := annotations[DbSecretAnnotation] + if !ok { + logrus.Error("Database Secret Update Event error! Annotation '", DbSecretAnnotation, "' value is empty or not exist.") + return + } + + logrus.Info("Processing Database Secret annotation: name=", DbSecretAnnotation, ", value=", dbSecretAnnotation) + + dbcrNames := strings.Split(dbSecretAnnotation, ",") + for _, dbcrName := range dbcrNames { + // send Database Reconcile Request + logrus.Info("Database Secret has been changed and related Database resource will be reconciled: secret=", secretNew.Namespace, "/", secretNew.Name, ", database=", dbcrName) + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: secretNew.GetNamespace(), + Name: dbcrName, + }}) + } + } +} + +func (e *secretEventHandler) Delete(event.DeleteEvent, workqueue.RateLimitingInterface) { + logrus.Error("secretEventHandler.Delete(...) event has been FIRED but NOT implemented!") +} +func (e *secretEventHandler) Generic(event.GenericEvent, workqueue.RateLimitingInterface) { + logrus.Error("secretEventHandler.Generic(...) event has been FIRED but NOT implemented!") +} +func (e *secretEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + logrus.Error("secretEventHandler.Create(...) event has been FIRED but NOT implemented!") +} + +/* ------ Event Filter Functions ------ */ + +func isWatchedNamespace(watchNamespaces []string, ro runtime.Object) bool { + if watchNamespaces[0] == "" { // # it's necessary to set "" to watch cluster wide + return true // watch for all namespaces + } + // define object's namespace + objectNamespace := "" + database, isDatabase := ro.(*kciv1alpha1.Database) + if isDatabase { + objectNamespace = database.Namespace + } else { + secret, isSecret := ro.(*corev1.Secret) + if isSecret { + objectNamespace = secret.Namespace + } else { + logrus.Info("unknown object", "object", ro) + return false + } + } + + // check that current namespace is watched by db-operator + for _, ns := range watchNamespaces { + if ns == objectNamespace { + return true + } + } + return false +} + +func isDatabase(ro runtime.Object) bool { + _, isDatabase := ro.(*kciv1alpha1.Database) + return isDatabase +} + +func isObjectUpdated(e event.UpdateEvent) bool { + if e.ObjectOld == nil { + logrus.Error(nil, "Update event has no old runtime object to update", "event", e) + return false + } + if e.ObjectNew == nil { + logrus.Error(nil, "Update event has no new runtime object for update", "event", e) + return false + } + // if object kind is a Database check that 'metadata.generation' field ('spec' section) has been changed + _, isDatabase := e.ObjectNew.(*kciv1alpha1.Database) + if isDatabase { + return e.ObjectNew.GetGeneration() != e.ObjectOld.GetGeneration() + } + + // if object kind is a Secret check that password value has changed + secretNew, isSecret := e.ObjectNew.(*corev1.Secret) + if isSecret { + // only annotated secrets are watched + annotations := secretNew.ObjectMeta.GetAnnotations() + dbcrName, ok := annotations[DbSecretAnnotation] + if !ok { + return false // no annotation found + } + logrus.Info("Secret Update Event detected: secret=", secretNew.Namespace, "/", secretNew.Name, ", database=", dbcrName) + return true + } + return false // unknown object, ignore Update Event +} diff --git a/controllers/helper.go b/controllers/helper.go index 8a61b3af..29363b42 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -19,27 +19,38 @@ package controllers import ( "context" + corev1 "k8s.io/api/core/v1" + kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1" "github.com/kloeckner-i/db-operator/pkg/utils/kci" crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) -func isDBSpecChanged(dbcr *kciv1alpha1.Database) bool { +func isDBChanged(dbcr *kciv1alpha1.Database, databaseSecret *corev1.Secret) bool { annotations := dbcr.ObjectMeta.GetAnnotations() - return annotations["checksum/spec"] != kci.GenerateChecksum(dbcr.Spec) + return annotations["checksum/spec"] != kci.GenerateChecksum(dbcr.Spec) || + annotations["checksum/secret"] != generateChecksumSecretValue(databaseSecret) } -func addDBSpecChecksum(dbcr *kciv1alpha1.Database) { +func addDBChecksum(dbcr *kciv1alpha1.Database, databaseSecret *corev1.Secret) { annotations := dbcr.ObjectMeta.GetAnnotations() if len(annotations) == 0 { annotations = make(map[string]string) } annotations["checksum/spec"] = kci.GenerateChecksum(dbcr.Spec) + annotations["checksum/secret"] = generateChecksumSecretValue(databaseSecret) dbcr.ObjectMeta.SetAnnotations(annotations) } +func generateChecksumSecretValue(databaseSecret *corev1.Secret) string { + if databaseSecret == nil || databaseSecret.Data == nil { + return "" + } + return kci.GenerateChecksum(databaseSecret.Data) +} + func isDBInstanceSpecChanged(ctx context.Context, dbin *kciv1alpha1.DbInstance) bool { checksums := dbin.Status.Checksums if checksums["spec"] != kci.GenerateChecksum(dbin.Spec) { diff --git a/controllers/helper_test.go b/controllers/helper_test.go index 8bfc5d84..ea26881f 100644 --- a/controllers/helper_test.go +++ b/controllers/helper_test.go @@ -31,6 +31,11 @@ import ( "k8s.io/apimachinery/pkg/types" ) +const ( + TestSecretName = "TestSec" + TestNamespace = "TestNS" +) + func newPostgresTestDbInstanceCr() kciv1alpha1.DbInstance { info := make(map[string]string) info["DB_PORT"] = "5432" @@ -50,8 +55,8 @@ func newPostgresTestDbInstanceCr() kciv1alpha1.DbInstance { } func newPostgresTestDbCr(instanceRef kciv1alpha1.DbInstance) *kciv1alpha1.Database { - o := metav1.ObjectMeta{Namespace: "TestNS"} - s := kciv1alpha1.DatabaseSpec{SecretName: "TestSec"} + o := metav1.ObjectMeta{Namespace: TestNamespace} + s := kciv1alpha1.DatabaseSpec{SecretName: TestSecretName} db := kciv1alpha1.Database{ ObjectMeta: o, @@ -94,13 +99,25 @@ func newMysqlTestDbCr() *kciv1alpha1.Database { } func TestIsSpecChanged(t *testing.T) { + db := newPostgresTestDbCr(newPostgresTestDbInstanceCr()) - addDBSpecChecksum(db) - nochange := isDBSpecChanged(db) + + testDbSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: TestNamespace, Name: TestSecretName}, + Data: map[string][]byte{ + "POSTGRES_DB": []byte("testdb"), + "POSTGRES_USER": []byte("testuser"), + "POSTGRES_PASSWORD": []byte("testpassword"), + }, + } + + addDBChecksum(db, testDbSecret) + nochange := isDBChanged(db, testDbSecret) assert.Equal(t, nochange, false, "expected false") - db.Spec.SecretName = "NewSec" - change := isDBSpecChanged(db) + testDbSecret.Data["POSTGRES_PASSWORD"] = []byte("testpasswordNEW") + + change := isDBChanged(db, testDbSecret) assert.Equal(t, change, true, "expected true") } diff --git a/integration/test.sh b/integration/test.sh index 3a9ec9cd..c3d18e8a 100755 --- a/integration/test.sh +++ b/integration/test.sh @@ -4,8 +4,8 @@ OPERATOR_NAMESPACE="operator" TEST_NAMESPACE="test" -retry=20 -interval=10 +retry=30 +interval=15 case $TEST_K8S in "microk8s") @@ -142,4 +142,4 @@ check_instance_status check_databases_status run_test delete_databases -check_databases_deleted \ No newline at end of file +check_databases_deleted diff --git a/main.go b/main.go index fe515fe7..aec7b720 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "flag" "os" "strconv" + "strings" "time" kcirocksv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1" @@ -101,6 +102,12 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "DbInstance") os.Exit(1) } + + + watchNamespaces := os.Getenv("WATCH_NAMESPACE") + namespaces := strings.Split(watchNamespaces, ",") + setupLog.Info("Database resources will be served in the next namespaces", "namespaces", namespaces) + if err = (&controllers.DatabaseReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("Database"), @@ -108,6 +115,7 @@ func main() { Recorder: mgr.GetEventRecorderFor("database-controller"), Interval: time.Duration(i), Conf: &conf, + WatchNamespaces: namespaces, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Database") os.Exit(1)