Skip to content

Commit

Permalink
Feature db operator watches for db passwords (#130)
Browse files Browse the repository at this point in the history
* feature-db-operator-watches-for-db-passwords

* remove empty rows

* add annotation "checksum/secret" to watch for DB secret updates

* fix database_controller.go

* remove empty rows

* fix helper.go

* fix requested changes

* introduce "watchNamespace" helm chart value

* fix helm chart version

* remove return statement

* add database iteration in the controllers/database_secret_handler.go

* introduce database secret annotation

* fix charts/db-operator/values.yaml

* fix annotation fetching

* remove func getDatabasesBySecret

* add logic to parse comma-separated "Database Secret" annotation value

* Update test.sh

* fix controllers/helper.go

* bump up appVersion in the charts/db-operator/Chart.yaml

Co-authored-by: 503095973 <[email protected]>
Co-authored-by: vpronkin <[email protected]>
  • Loading branch information
3 people committed Mar 17, 2022
1 parent fa098fe commit 38f34e6
Show file tree
Hide file tree
Showing 10 changed files with 253 additions and 24 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-and-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ jobs:
needs: unit-test
strategy:
matrix:
k8s_version: ['v1.19.12', 'v1.20.12', 'v1.21.6', 'v1.22.3']
k8s_version: ['v1.19.12', 'v1.20.12', 'v1.21.6', 'v1.22.3', 'v1.23.1']
steps:
- name: Checkout
uses: actions/checkout@v2
Expand Down
6 changes: 3 additions & 3 deletions charts/db-operator/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: v2
type: application
kubeVersion: ">= 1.19-prerelease <= 1.22-prerelease"
appVersion: "1.4.0"
kubeVersion: ">= 1.19-prerelease <= 1.23-prerelease"
appVersion: "1.5.0"
description: A Database Operator
name: db-operator
version: 1.1.4
version: 1.1.5
2 changes: 1 addition & 1 deletion charts/db-operator/templates/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ spec:
- name: OPERATOR_NAME
value: "db-operator"
- name: WATCH_NAMESPACE
value: "" # it's necessary to set "" to watch cluster wide
value: {{ .Values.watchNamespace | quote }}
- name: RECONCILE_INTERVAL
value: {{ .Values.reconcileInterval | quote }}
- name: POD_NAME
Expand Down
2 changes: 2 additions & 0 deletions charts/db-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ image:
# - name: myRegistrySecret

reconcileInterval: "60"
# watchNamespace value is comma-separated list of namespace names. It's necessary to set "" to watch cluster wide.
watchNamespace: ""

rbac:
create: true
Expand Down
57 changes: 50 additions & 7 deletions controllers/database_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ import (
"errors"
"io/ioutil"
"os"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/source"
"strconv"
"time"

Expand Down Expand Up @@ -48,11 +51,12 @@ import (
// DatabaseReconciler reconciles a Database object
type DatabaseReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
Recorder record.EventRecorder
Interval time.Duration
Conf *config.Config
Log logr.Logger
Scheme *runtime.Scheme
Recorder record.EventRecorder
Interval time.Duration
Conf *config.Config
WatchNamespaces []string
}

var (
Expand Down Expand Up @@ -150,7 +154,13 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
return reconcileResult, nil
}

if isDBSpecChanged(dbcr) {
databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
if err != nil && !k8serrors.IsNotFound(err) {
logrus.Errorf("could not get database secret - %s", err)
return r.manageError(ctx, dbcr, err, true)
}

if isDBChanged(dbcr, databaseSecret) {
logrus.Infof("DB: namespace=%s, name=%s spec changed", dbcr.Namespace, dbcr.Name)
err := r.initialize(ctx, dbcr)
if err != nil {
Expand All @@ -162,7 +172,7 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
return r.manageError(ctx, dbcr, err, true)
}

addDBSpecChecksum(dbcr)
addDBChecksum(dbcr, databaseSecret)
err = r.Update(ctx, dbcr)
if err != nil {
logrus.Errorf("error resource updating - %s", err)
Expand Down Expand Up @@ -248,8 +258,24 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c

// SetupWithManager sets up the controller with the Manager.
func (r *DatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {

eventFilter := predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
return isWatchedNamespace(r.WatchNamespaces, e.Object) && isDatabase(e.Object)
}, // Reconcile only Database Create Event
DeleteFunc: func(e event.DeleteEvent) bool {
return isWatchedNamespace(r.WatchNamespaces, e.Object) && isDatabase(e.Object)
}, // Reconcile only Database Delete Event
UpdateFunc: func(e event.UpdateEvent) bool {
return isWatchedNamespace(r.WatchNamespaces, e.ObjectNew) && isObjectUpdated(e)
}, // Reconcile Database and Secret Update Events
GenericFunc: func(e event.GenericEvent) bool { return true }, // Reconcile any Generic Events (operator POD or cluster restarted)
}

return ctrl.NewControllerManagedBy(mgr).
For(&kciv1alpha1.Database{}).
WithEventFilter(eventFilter).
Watches(&source.Kind{Type: &corev1.Secret{}}, &secretEventHandler{r.Client}).
Complete(r)
}

Expand Down Expand Up @@ -342,6 +368,12 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph
return err
}

err = r.annotateDatabaseSecret(ctx, dbcr, databaseSecret)
if err != nil {
logrus.Errorf("could not annotate database secret - %s", err)
return err
}

dbcr.Status.DatabaseName = databaseCred.Name
dbcr.Status.UserName = databaseCred.Username
logrus.Infof("DB: namespace=%s, name=%s successfully created", dbcr.Namespace, dbcr.Name)
Expand Down Expand Up @@ -677,6 +709,17 @@ func (r *DatabaseReconciler) getDatabaseSecret(ctx context.Context, dbcr *kciv1a
return secret, nil
}

func (r *DatabaseReconciler) annotateDatabaseSecret(ctx context.Context, dbcr *kciv1alpha1.Database, secret *corev1.Secret) error {
annotations := secret.ObjectMeta.GetAnnotations()
if len(annotations) == 0 {
annotations = make(map[string]string)
}
annotations[DbSecretAnnotation] = dbcr.Name
secret.ObjectMeta.SetAnnotations(annotations)

return r.Update(ctx, secret)
}

func (r *DatabaseReconciler) getAdminSecret(ctx context.Context, dbcr *kciv1alpha1.Database) (*corev1.Secret, error) {
instance, err := dbcr.GetInstanceRef()
if err != nil {
Expand Down
148 changes: 148 additions & 0 deletions controllers/database_secret_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
* Copyright 2021 kloeckner.i GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package controllers

import (
kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"strings"
)

const (
DbSecretAnnotation = "db-operator/database"
)

/* ------ Secret Event Handler ------ */
type secretEventHandler struct {
client.Client
}

func (e *secretEventHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {

logrus.Info("Start processing Database Secret Update Event")

switch v := evt.ObjectNew.(type) {

default:
logrus.Error("Database Secret Update Event error! Unknown object: type=", v.GetObjectKind(), ", name=", evt.ObjectNew.GetNamespace(), "/", evt.ObjectNew.GetName())
return

case *corev1.Secret:
// only annotated secrets are watched
secretNew := evt.ObjectNew.(*corev1.Secret)
annotations := secretNew.ObjectMeta.GetAnnotations()
dbSecretAnnotation, ok := annotations[DbSecretAnnotation]
if !ok {
logrus.Error("Database Secret Update Event error! Annotation '", DbSecretAnnotation, "' value is empty or not exist.")
return
}

logrus.Info("Processing Database Secret annotation: name=", DbSecretAnnotation, ", value=", dbSecretAnnotation)

dbcrNames := strings.Split(dbSecretAnnotation, ",")
for _, dbcrName := range dbcrNames {
// send Database Reconcile Request
logrus.Info("Database Secret has been changed and related Database resource will be reconciled: secret=", secretNew.Namespace, "/", secretNew.Name, ", database=", dbcrName)
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Namespace: secretNew.GetNamespace(),
Name: dbcrName,
}})
}
}
}

func (e *secretEventHandler) Delete(event.DeleteEvent, workqueue.RateLimitingInterface) {
logrus.Error("secretEventHandler.Delete(...) event has been FIRED but NOT implemented!")
}
func (e *secretEventHandler) Generic(event.GenericEvent, workqueue.RateLimitingInterface) {
logrus.Error("secretEventHandler.Generic(...) event has been FIRED but NOT implemented!")
}
func (e *secretEventHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
logrus.Error("secretEventHandler.Create(...) event has been FIRED but NOT implemented!")
}

/* ------ Event Filter Functions ------ */

func isWatchedNamespace(watchNamespaces []string, ro runtime.Object) bool {
if watchNamespaces[0] == "" { // # it's necessary to set "" to watch cluster wide
return true // watch for all namespaces
}
// define object's namespace
objectNamespace := ""
database, isDatabase := ro.(*kciv1alpha1.Database)
if isDatabase {
objectNamespace = database.Namespace
} else {
secret, isSecret := ro.(*corev1.Secret)
if isSecret {
objectNamespace = secret.Namespace
} else {
logrus.Info("unknown object", "object", ro)
return false
}
}

// check that current namespace is watched by db-operator
for _, ns := range watchNamespaces {
if ns == objectNamespace {
return true
}
}
return false
}

func isDatabase(ro runtime.Object) bool {
_, isDatabase := ro.(*kciv1alpha1.Database)
return isDatabase
}

func isObjectUpdated(e event.UpdateEvent) bool {
if e.ObjectOld == nil {
logrus.Error(nil, "Update event has no old runtime object to update", "event", e)
return false
}
if e.ObjectNew == nil {
logrus.Error(nil, "Update event has no new runtime object for update", "event", e)
return false
}
// if object kind is a Database check that 'metadata.generation' field ('spec' section) has been changed
_, isDatabase := e.ObjectNew.(*kciv1alpha1.Database)
if isDatabase {
return e.ObjectNew.GetGeneration() != e.ObjectOld.GetGeneration()
}

// if object kind is a Secret check that password value has changed
secretNew, isSecret := e.ObjectNew.(*corev1.Secret)
if isSecret {
// only annotated secrets are watched
annotations := secretNew.ObjectMeta.GetAnnotations()
dbcrName, ok := annotations[DbSecretAnnotation]
if !ok {
return false // no annotation found
}
logrus.Info("Secret Update Event detected: secret=", secretNew.Namespace, "/", secretNew.Name, ", database=", dbcrName)
return true
}
return false // unknown object, ignore Update Event
}
17 changes: 14 additions & 3 deletions controllers/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,38 @@ package controllers
import (
"context"

corev1 "k8s.io/api/core/v1"

kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1"
"github.com/kloeckner-i/db-operator/pkg/utils/kci"
crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
)

func isDBSpecChanged(dbcr *kciv1alpha1.Database) bool {
func isDBChanged(dbcr *kciv1alpha1.Database, databaseSecret *corev1.Secret) bool {
annotations := dbcr.ObjectMeta.GetAnnotations()

return annotations["checksum/spec"] != kci.GenerateChecksum(dbcr.Spec)
return annotations["checksum/spec"] != kci.GenerateChecksum(dbcr.Spec) ||
annotations["checksum/secret"] != generateChecksumSecretValue(databaseSecret)
}

func addDBSpecChecksum(dbcr *kciv1alpha1.Database) {
func addDBChecksum(dbcr *kciv1alpha1.Database, databaseSecret *corev1.Secret) {
annotations := dbcr.ObjectMeta.GetAnnotations()
if len(annotations) == 0 {
annotations = make(map[string]string)
}

annotations["checksum/spec"] = kci.GenerateChecksum(dbcr.Spec)
annotations["checksum/secret"] = generateChecksumSecretValue(databaseSecret)
dbcr.ObjectMeta.SetAnnotations(annotations)
}

func generateChecksumSecretValue(databaseSecret *corev1.Secret) string {
if databaseSecret == nil || databaseSecret.Data == nil {
return ""
}
return kci.GenerateChecksum(databaseSecret.Data)
}

func isDBInstanceSpecChanged(ctx context.Context, dbin *kciv1alpha1.DbInstance) bool {
checksums := dbin.Status.Checksums
if checksums["spec"] != kci.GenerateChecksum(dbin.Spec) {
Expand Down
29 changes: 23 additions & 6 deletions controllers/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ import (
"k8s.io/apimachinery/pkg/types"
)

const (
TestSecretName = "TestSec"
TestNamespace = "TestNS"
)

func newPostgresTestDbInstanceCr() kciv1alpha1.DbInstance {
info := make(map[string]string)
info["DB_PORT"] = "5432"
Expand All @@ -50,8 +55,8 @@ func newPostgresTestDbInstanceCr() kciv1alpha1.DbInstance {
}

func newPostgresTestDbCr(instanceRef kciv1alpha1.DbInstance) *kciv1alpha1.Database {
o := metav1.ObjectMeta{Namespace: "TestNS"}
s := kciv1alpha1.DatabaseSpec{SecretName: "TestSec"}
o := metav1.ObjectMeta{Namespace: TestNamespace}
s := kciv1alpha1.DatabaseSpec{SecretName: TestSecretName}

db := kciv1alpha1.Database{
ObjectMeta: o,
Expand Down Expand Up @@ -94,13 +99,25 @@ func newMysqlTestDbCr() *kciv1alpha1.Database {
}

func TestIsSpecChanged(t *testing.T) {

db := newPostgresTestDbCr(newPostgresTestDbInstanceCr())
addDBSpecChecksum(db)
nochange := isDBSpecChanged(db)

testDbSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: TestNamespace, Name: TestSecretName},
Data: map[string][]byte{
"POSTGRES_DB": []byte("testdb"),
"POSTGRES_USER": []byte("testuser"),
"POSTGRES_PASSWORD": []byte("testpassword"),
},
}

addDBChecksum(db, testDbSecret)
nochange := isDBChanged(db, testDbSecret)
assert.Equal(t, nochange, false, "expected false")

db.Spec.SecretName = "NewSec"
change := isDBSpecChanged(db)
testDbSecret.Data["POSTGRES_PASSWORD"] = []byte("testpasswordNEW")

change := isDBChanged(db, testDbSecret)
assert.Equal(t, change, true, "expected true")
}

Expand Down
Loading

0 comments on commit 38f34e6

Please sign in to comment.