From ed440c4cedd79d435ce25641af5200f684d75772 Mon Sep 17 00:00:00 2001 From: Nikolai Rodionov Date: Wed, 16 Nov 2022 16:52:48 +0100 Subject: [PATCH] Support templating generated secret (#161) --- README.md | 4 +- api/v1alpha1/database_types.go | 5 +- api/v1alpha1/zz_generated.deepcopy.go | 29 +++ config/crd/bases/kci.rocks_databases.yaml | 4 + controllers/database_controller.go | 64 +++++-- controllers/database_helper.go | 151 ++++++++++++++- controllers/database_helper_test.go | 213 +++++++++++++++++++++- docs/creatingdatabases.md | 15 +- go.mod | 3 +- go.sum | 3 - pkg/utils/database/types.go | 2 +- 11 files changed, 446 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 0c652076..69638000 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,9 @@ $ make k3s_mac_image ### Deploy ``` -helm upgrade my-release kloeckneri/db-operator --set image.repository=my-db-operator --set image.tag=1.0.0-dev --set image.pullPolicy=IfNotPresent +helm repo add kloeckneri https://kloeckner-i.github.io/charts +helm repo update +helm upgrade my-release kloeckneri/db-operator --set image.repository=my-db-operator --set image.tag=1.0.0-dev --set image.pullPolicy=IfNotPresent --install ``` ### Run unit test locally diff --git a/api/v1alpha1/database_types.go b/api/v1alpha1/database_types.go index 7ed46d0c..c6716020 100644 --- a/api/v1alpha1/database_types.go +++ b/api/v1alpha1/database_types.go @@ -35,8 +35,9 @@ type DatabaseSpec struct { // These keywords can be used: Protocol, DatabaseHost, DatabasePort, UserName, Password, DatabaseName. // Default template looks like this: // "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" - ConnectionStringTemplate string `json:"connectionStringTemplate,omitempty"` - Postgres Postgres `json:"postgres,omitempty"` + ConnectionStringTemplate string `json:"connectionStringTemplate,omitempty"` + SecretsTemplates map[string]string `json:"secretsTemplates,omitempty"` + Postgres Postgres `json:"postgres,omitempty"` } // Postgres struct should be used to provide resource that only applicable to postgres diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d2852f17..57407d56 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -138,6 +138,14 @@ func (in *DatabaseSpec) DeepCopyInto(out *DatabaseSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.SecretsTemplates != nil { + in, out := &in.SecretsTemplates, &out.SecretsTemplates + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Postgres.DeepCopyInto(&out.Postgres) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatabaseSpec. @@ -368,6 +376,7 @@ func (in *GenericInstance) DeepCopy() *GenericInstance { func (in *GoogleInstance) DeepCopyInto(out *GoogleInstance) { *out = *in out.ConfigmapName = in.ConfigmapName + out.ClientSecret = in.ClientSecret } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GoogleInstance. @@ -394,3 +403,23 @@ func (in *NamespacedName) DeepCopy() *NamespacedName { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Postgres) DeepCopyInto(out *Postgres) { + *out = *in + if in.Schemas != nil { + in, out := &in.Schemas, &out.Schemas + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Postgres. +func (in *Postgres) DeepCopy() *Postgres { + if in == nil { + return nil + } + out := new(Postgres) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/kci.rocks_databases.yaml b/config/crd/bases/kci.rocks_databases.yaml index e0cbca56..9c5b14f3 100644 --- a/config/crd/bases/kci.rocks_databases.yaml +++ b/config/crd/bases/kci.rocks_databases.yaml @@ -104,6 +104,10 @@ spec: type: object secretName: type: string + secretsTemplates: + additionalProperties: + type: string + type: object required: - backup - deletionProtected diff --git a/controllers/database_controller.go b/controllers/database_controller.go index 4109a579..6c66e769 100644 --- a/controllers/database_controller.go +++ b/controllers/database_controller.go @@ -63,7 +63,7 @@ var ( dbPhaseCreate = "Creating" dbPhaseInstanceAccessSecret = "InstanceAccessSecretCreating" dbPhaseProxy = "ProxyCreating" - dbPhaseConnectionString = "ConnectionStringCreating" + dbPhaseSecretsTemplating = "SecretsTemplating" dbPhaseConfigMap = "InfoConfigMapCreating" dbPhaseMonitoring = "MonitoringCreating" dbPhaseBackupJob = "BackupJobCreating" @@ -124,7 +124,6 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c // finalization logic fails, don't remove the finalizer so // that we can retry during the next reconciliation. if containsString(dbcr.ObjectMeta.Finalizers, "db."+dbcr.Name) { - logrus.Infof("DB: namespace=%s, name=%s deleting database", dbcr.Namespace, dbcr.Name) err := r.deleteDatabase(ctx, dbcr) if err != nil { logrus.Errorf("DB: namespace=%s, name=%s failed deleting database - %s", dbcr.Namespace, dbcr.Name, err) @@ -205,9 +204,9 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err != nil { return r.manageError(ctx, dbcr, err, true) } - dbcr.Status.Phase = dbPhaseConnectionString - case dbPhaseConnectionString: - err := r.createConnectionString(ctx, dbcr) + dbcr.Status.Phase = dbPhaseSecretsTemplating + case dbPhaseSecretsTemplating: + err := r.createTemplatedSecrets(ctx, dbcr) if err != nil { return r.manageError(ctx, dbcr, err, true) } @@ -352,7 +351,6 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph err = database.Create(db, adminCred) if err != nil { - return err } @@ -591,34 +589,68 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1. return nil } -func (r *DatabaseReconciler) createConnectionString(ctx context.Context, dbcr *kciv1alpha1.Database) error { +func (r *DatabaseReconciler) createTemplatedSecrets(ctx context.Context, dbcr *kciv1alpha1.Database) error { // First of all the password should be taken from secret because it's not stored anywhere else databaseSecret, err := r.getDatabaseSecret(ctx, dbcr) if err != nil { return err } // Then parse the secret to get the password - databaseCred, err := parseDatabaseSecretData(dbcr, databaseSecret.Data) + // Connection stirng is deprecated and will be removed soon. So this switch is temporary. + // Once connection string is removed, the switch and the following if condition are gone + useLegacyConnectionString := false + switch { + case len(dbcr.Spec.ConnectionStringTemplate) > 0 && len(dbcr.Spec.SecretsTemplates) > 0: + logrus.Warnf("DB: namespace=%s, name=%s connectionStringTemplate will be ignored since secretsTemplates is not empty", + dbcr.Namespace, + dbcr.Name, + ) + case len(dbcr.Spec.ConnectionStringTemplate) > 0: + logrus.Warnf("DB: namespace=%s, name=%s connectionStringTemplate is deprecated and will be removed in the near future, consider using secretsTemplates", + dbcr.Namespace, + dbcr.Name, + ) + useLegacyConnectionString = true + default: + logrus.Infof("DB: namespace=%s, name=%s generating secrets", dbcr.Namespace, dbcr.Name) + } + + databaseCred, err := parseTemplatedSecretsData(dbcr, databaseSecret.Data, useLegacyConnectionString) if err != nil { return err } - // Generate the connection string - dbConnectionString, err := generateConnectionString(dbcr, databaseCred) + if useLegacyConnectionString { + // Generate the connection string + dbConnectionString, err := generateConnectionString(dbcr, databaseCred) + if err != nil { + return err + } + // Update database-credentials secret. + if databaseCred.TemplatedSecrets["CONNECTION_STRING"] == dbConnectionString { + return nil + } + logrus.Debugf("DB: namespace=%s, name=%s updating credentials secret", dbcr.Namespace, dbcr.Name) + newSecret := addConnectionStringToSecret(dbcr, databaseSecret.Data, dbConnectionString) + return r.Update(ctx, newSecret, &client.UpdateOptions{}) + } + + dbSecrets, err := generateTemplatedSecrets(dbcr, databaseCred) if err != nil { return err } - // Update database-credentials secret. - if databaseCred.ConnectionString == dbConnectionString { - return nil + // Adding values + newSecret := fillTemplatedSecretData(dbcr, databaseSecret.Data, dbSecrets) + err = r.Update(ctx, newSecret, &client.UpdateOptions{}) + if err != nil { + return err } - logrus.Debugf("DB: namespace=%s, name=%s updating credentials secret", dbcr.Namespace, dbcr.Name) - newSecret := addConnectionStringToSecret(dbcr, databaseSecret.Data, dbConnectionString) + newSecret = removeObsoleteSecret(dbcr, databaseSecret.Data, dbSecrets) err = r.Update(ctx, newSecret, &client.UpdateOptions{}) if err != nil { return err } - logrus.Infof("DB: namespace=%s, name=%s connection string is added to credentials secret", dbcr.Namespace, dbcr.Name) + return nil } diff --git a/controllers/database_helper.go b/controllers/database_helper.go index bd9c15cd..b4c983be 100644 --- a/controllers/database_helper.go +++ b/controllers/database_helper.go @@ -27,10 +27,11 @@ import ( "github.com/kloeckner-i/db-operator/pkg/utils/kci" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + "k8s.io/utils/strings/slices" ) -// ConnectionStringFields defines default fields that can be used to generate a connection string -type ConnectionStringFields struct { +// SecretsTemplatesFields defines default fields that can be used to generate secrets with db creds +type SecretsTemplatesFields struct { Protocol string DatabaseHost string DatabasePort int32 @@ -39,6 +40,19 @@ type ConnectionStringFields struct { DatabaseName string } +const ( + fieldPostgresDB = "POSTGRES_DB" + fieldPostgresUser = "POSTGRES_USER" + fieldPostgressPassword = "POSTGRES_PASSWORD" + fieldMysqlDB = "DB" + fieldMysqlUser = "USER" + fieldMysqlPassword = "PASSWORD" +) + +func getBlockedTempatedKeys() []string { + return []string{fieldMysqlDB, fieldMysqlPassword, fieldMysqlUser, fieldPostgresDB, fieldPostgresUser, fieldPostgressPassword} +} + func determinDatabaseType(dbcr *kciv1alpha1.Database, dbCred database.Credentials) (database.Database, error) { instance, err := dbcr.GetInstanceRef() if err != nil { @@ -110,18 +124,43 @@ func determinDatabaseType(dbcr *kciv1alpha1.Database, dbCred database.Credential } } -func parseDatabaseSecretData(dbcr *kciv1alpha1.Database, data map[string][]byte) (database.Credentials, error) { - cred := database.Credentials{} - engine, err := dbcr.GetEngineType() +func parseTemplatedSecretsData(dbcr *kciv1alpha1.Database, data map[string][]byte, useLegacyConnStr bool) (database.Credentials, error) { + cred, err := parseDatabaseSecretData(dbcr, data) if err != nil { return cred, err } + cred.TemplatedSecrets = map[string]string{} - // Connection string can be empty - if connectionString, ok := data["CONNECTION_STRING"]; ok { - cred.ConnectionString = string(connectionString) + if useLegacyConnStr { + if connectionString, ok := data["CONNECTION_STRING"]; ok { + cred.TemplatedSecrets["CONNECTION_STRING"] = string(connectionString) + } else { + logrus.Infof("DB: namespace=%s, name=%s CONNECTION_STRING key does not exist in the secret data", dbcr.Namespace, dbcr.Name) + } } else { - logrus.Info("CONNECTION_STRING key does not exist in secret data") + for key := range dbcr.Spec.SecretsTemplates { + // Here we can see if there are obsolete entries in the secret data + if secret, ok := data[key]; ok { + delete(data, key) + cred.TemplatedSecrets[key] = string(secret) + } else { + logrus.Infof("DB: namespace=%s, name=%s %s key does not exist in secret data", + dbcr.Namespace, + dbcr.Name, + key, + ) + } + } + } + + return cred, nil +} + +func parseDatabaseSecretData(dbcr *kciv1alpha1.Database, data map[string][]byte) (database.Credentials, error) { + cred := database.Credentials{} + engine, err := dbcr.GetEngineType() + if err != nil { + return cred, err } switch engine { @@ -211,7 +250,7 @@ func generateConnectionString(dbcr *kciv1alpha1.Database, databaseCred database. // "postgresql://user:password@host:port/database" const defaultTemplate = "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" - dbData := ConnectionStringFields{ + dbData := SecretsTemplatesFields{ DatabaseHost: dbcr.Status.ProxyStatus.ServiceName, DatabasePort: dbcr.Status.ProxyStatus.SQLPort, UserName: databaseCred.Username, @@ -262,7 +301,99 @@ func generateConnectionString(dbcr *kciv1alpha1.Database, databaseCred database. return } +func generateTemplatedSecrets(dbcr *kciv1alpha1.Database, databaseCred database.Credentials) (secrets map[string]string, err error) { + secrets = map[string]string{} + templates := map[string]string{} + if len(dbcr.Spec.SecretsTemplates) > 0 { + templates = dbcr.Spec.SecretsTemplates + } else { + const tmpl = "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" + templates["CONNECTION_STRING"] = tmpl + } + // The string that's going to be generated if the default template is used: + // "postgresql://user:password@host:port/database" + dbData := SecretsTemplatesFields{ + DatabaseHost: dbcr.Status.ProxyStatus.ServiceName, + DatabasePort: dbcr.Status.ProxyStatus.SQLPort, + UserName: databaseCred.Username, + Password: databaseCred.Password, + DatabaseName: databaseCred.Name, + } + + // If proxy is not used, set a real database address + if !dbcr.Status.ProxyStatus.Status { + db, err := determinDatabaseType(dbcr, databaseCred) + if err != nil { + return nil, err + } + dbAddress := db.GetDatabaseAddress() + dbData.DatabaseHost = dbAddress.Host + dbData.DatabasePort = int32(dbAddress.Port) + } + // If engine is 'postgres', the protocol should be postgresql + if dbcr.Status.InstanceRef.Spec.Engine == "postgres" { + dbData.Protocol = "postgresql" + } else { + dbData.Protocol = dbcr.Status.InstanceRef.Spec.Engine + } + + logrus.Infof("DB: namespace=%s, name=%s creating secrets from templates", dbcr.Namespace, dbcr.Name) + for key, value := range templates { + var tmpl string = value + t, err := template.New("secret").Parse(tmpl) + if err != nil { + return nil, err + } + + var secretBytes bytes.Buffer + err = t.Execute(&secretBytes, dbData) + if err != nil { + return nil, err + } + connString := secretBytes.String() + secrets[key] = connString + } + return secrets, nil +} + +func fillTemplatedSecretData(dbcr *kciv1alpha1.Database, secretData map[string][]byte, newSecretFields map[string]string) (newSecret *v1.Secret) { + blockedTempatedKeys := getBlockedTempatedKeys() + for key, value := range newSecretFields { + if slices.Contains(blockedTempatedKeys, key) { + logrus.Warnf("DB: namespace=%s, name=%s %s can't be used for templating, because it's used for default secret created by operator", + dbcr.Namespace, + dbcr.Name, + key, + ) + } else { + newSecret = addTemplatedSecretToSecret(dbcr, secretData, key, value) + } + } + return +} + func addConnectionStringToSecret(dbcr *kciv1alpha1.Database, secretData map[string][]byte, connectionString string) *v1.Secret { secretData["CONNECTION_STRING"] = []byte(connectionString) return kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.GetNamespace(), secretData) } + +func addTemplatedSecretToSecret(dbcr *kciv1alpha1.Database, secretData map[string][]byte, secretName string, secretValue string) *v1.Secret { + secretData[secretName] = []byte(secretValue) + return kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.GetNamespace(), secretData) +} + +func removeObsoleteSecret(dbcr *kciv1alpha1.Database, secretData map[string][]byte, newSecretFields map[string]string) *v1.Secret { + blockedTempatedKeys := getBlockedTempatedKeys() + + for key := range secretData { + if _, ok := newSecretFields[key]; !ok { + // Check if is a untemplatead secret, so it's not removed accidentally + if !slices.Contains(blockedTempatedKeys, key) { + logrus.Infof("DB: namespace=%s, name=%s removing an obsolete field: %s", dbcr.Namespace, dbcr.Name, key) + delete(secretData, key) + } + } + } + + return kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.GetNamespace(), secretData) +} diff --git a/controllers/database_helper_test.go b/controllers/database_helper_test.go index 5426111b..7ca6917a 100644 --- a/controllers/database_helper_test.go +++ b/controllers/database_helper_test.go @@ -22,6 +22,7 @@ import ( "github.com/kloeckner-i/db-operator/pkg/utils/database" "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" ) var testDbcred = database.Credentials{Name: "testdb", Username: "testuser", Password: "password"} @@ -113,12 +114,214 @@ func TestMonitoringEnabled(t *testing.T) { assert.Equal(t, postgresInterface.Monitoring, true, "expected monitoring is true in postgres interface") } +func TestPsqlDefaultTemplatedSecretGeneratationWithProxy(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + postgresDbCr.Status.ProxyStatus.Status = true + + c := SecretsTemplatesFields{ + DatabaseHost: "postgres", + DatabasePort: 5432, + UserName: testDbcred.Username, + Password: testDbcred.Password, + DatabaseName: testDbcred.Name, + } + + postgresDbCr.Status.ProxyStatus.SQLPort = c.DatabasePort + postgresDbCr.Status.ProxyStatus.ServiceName = c.DatabaseHost + + protocol := "postgresql" + expectedData := map[string]string{ + "CONNECTION_STRING": fmt.Sprintf("%s://%s:%s@%s:%d/%s", protocol, c.UserName, c.Password, c.DatabaseHost, c.DatabasePort, c.DatabaseName), + } + + connString, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + if err != nil { + t.Logf("Unexpected error: %s", err) + t.Fail() + } + assert.Equal(t, expectedData, connString, "generated connections string is wrong") +} + +func TestPsqlDefaultTemplatedSecretGeneratationWithoutProxy(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + + c := SecretsTemplatesFields{ + DatabaseHost: "postgres", + DatabasePort: 5432, + UserName: testDbcred.Username, + Password: testDbcred.Password, + DatabaseName: testDbcred.Name, + } + + protocol := "postgresql" + expectedData := map[string]string{ + "CONNECTION_STRING": fmt.Sprintf("%s://%s:%s@%s:%d/%s", protocol, c.UserName, c.Password, c.DatabaseHost, c.DatabasePort, c.DatabaseName), + } + + connString, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + if err != nil { + t.Logf("Unexpected error: %s", err) + t.Fail() + } + assert.Equal(t, expectedData, connString, "generated connections string is wrong") +} + +func TestMysqlDefaultTemlatedSecretGeneratationWithoutProxy(t *testing.T) { + mysqlDbCr := newMysqlTestDbCr() + c := SecretsTemplatesFields{ + DatabaseHost: "mysql", + DatabasePort: 3306, + UserName: testDbcred.Username, + Password: testDbcred.Password, + DatabaseName: testDbcred.Name, + } + protocol := "mysql" + expectedData := map[string]string{ + "CONNECTION_STRING": fmt.Sprintf("%s://%s:%s@%s:%d/%s", protocol, c.UserName, c.Password, c.DatabaseHost, c.DatabasePort, c.DatabaseName), + } + + connString, err := generateTemplatedSecrets(mysqlDbCr, testDbcred) + if err != nil { + t.Logf("Unexpected error: %s", err) + t.Fail() + } + assert.Equal(t, connString, expectedData, "generated connections string is wrong") +} + +func TestAddingTemplatedSecretsToSecret(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + secretData := map[string][]byte{ + "POSTGRES_DB": []byte("postgres"), + "POSTGRES_USER": []byte("root"), + "POSTGRES_PASSWORD": []byte("qwertyu9"), + } + + connectionString := "it's a dummy connection string" + + secret := addTemplatedSecretToSecret(postgresDbCr, secretData, "TMPL", connectionString) + secretData["CONNECTION_STRING"] = []byte(connectionString) + if val, ok := secret.Data["TMPL"]; ok { + assert.Equal(t, string(val), connectionString, "connections string in a secret contains unexpected values") + return + } +} + +func TestPsqlCustomSecretGeneratation(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + + prefix := "custom->" + postfix := "<-for_storing_data_you_know" + postgresDbCr.Spec.SecretsTemplates = map[string]string{ + "CHECK_1": fmt.Sprintf("%s{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}%s", prefix, postfix), + "CHECK_2": "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}", + } + + c := SecretsTemplatesFields{ + DatabaseHost: "postgres", + DatabasePort: 5432, + UserName: testDbcred.Username, + Password: testDbcred.Password, + DatabaseName: testDbcred.Name, + } + protocol := "postgresql" + expectedData := map[string]string{ + "CHECK_1": fmt.Sprintf("%s%s://%s:%s@%s:%d/%s%s", prefix, protocol, c.UserName, c.Password, c.DatabaseHost, c.DatabasePort, c.DatabaseName, postfix), + "CHECK_2": fmt.Sprintf("%s://%s:%s@%s:%d/%s", protocol, c.UserName, c.Password, c.DatabaseHost, c.DatabasePort, c.DatabaseName), + } + + connString, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + if err != nil { + t.Logf("unexpected error: %s", err) + t.Fail() + } + assert.Equal(t, connString, expectedData, "generated connections string is wrong") +} + +func TestWrongTemplatedSecretGeneratation(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + + postgresDbCr.Spec.SecretsTemplates = map[string]string{ + "TMPL": "{{ .Protocol }}://{{ .User }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}", + } + + _, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + errSubstr := "can't evaluate field User in type controllers.SecretsTemplatesFields" + + assert.Contains(t, err.Error(), errSubstr, "the error doesn't contain expected substring") +} + +func TestBlockedTempatedKeysGeneratation(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + + postgresDbCr.Spec.SecretsTemplates = map[string]string{} + untemplatedFields := []string{fieldMysqlDB, fieldMysqlPassword, fieldMysqlUser, fieldPostgresDB, fieldPostgresUser, fieldPostgressPassword} + for _, key := range untemplatedFields { + postgresDbCr.Spec.SecretsTemplates[key] = "DUMMY" + } + postgresDbCr.Spec.SecretsTemplates["TMPL"] = "DUMMY" + expectedData := map[string][]byte{ + "TMPL": []byte("DUMMY"), + } + + sercretData, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + if err != nil { + t.Logf("unexpected error: %s", err) + t.Fail() + } + + dummySecret := v1.Secret{ + Data: map[string][]byte{}, + } + + newSecret := fillTemplatedSecretData(postgresDbCr, dummySecret.Data, sercretData) + assert.Equal(t, newSecret.Data, expectedData, "generated connections string is wrong") +} + +func TestObsoleteFieldsRemoving(t *testing.T) { + instance := newPostgresTestDbInstanceCr() + postgresDbCr := newPostgresTestDbCr(instance) + + postgresDbCr.Spec.SecretsTemplates = map[string]string{} + untemplatedFields := []string{fieldMysqlDB, fieldMysqlPassword, fieldMysqlUser, fieldPostgresDB, fieldPostgresUser, fieldPostgressPassword} + for _, key := range untemplatedFields { + postgresDbCr.Spec.SecretsTemplates[key] = "DUMMY" + } + postgresDbCr.Spec.SecretsTemplates["TMPL"] = "DUMMY" + expectedData := map[string][]byte{ + "TMPL": []byte("DUMMY"), + } + + sercretData, err := generateTemplatedSecrets(postgresDbCr, testDbcred) + if err != nil { + t.Logf("unexpected error: %s", err) + t.Fail() + } + + dummySecret := v1.Secret{ + Data: map[string][]byte{ + "TO_REMOVE": []byte("this is supposed to be removed"), + }, + } + + newSecret := fillTemplatedSecretData(postgresDbCr, dummySecret.Data, sercretData) + newSecret = removeObsoleteSecret(postgresDbCr, dummySecret.Data, sercretData) + + assert.Equal(t, newSecret.Data, expectedData, "generated connections string is wrong") +} + +// Connection string tests should be removed later, when connection string is gone func TestPsqlDefaultConnectionStringGeneratationWithProxy(t *testing.T) { instance := newPostgresTestDbInstanceCr() postgresDbCr := newPostgresTestDbCr(instance) postgresDbCr.Status.ProxyStatus.Status = true - c := ConnectionStringFields{ + c := SecretsTemplatesFields{ DatabaseHost: "postgres", DatabasePort: 5432, UserName: testDbcred.Username, @@ -144,7 +347,7 @@ func TestPsqlDefaultConnectionStringGeneratationWithoutProxy(t *testing.T) { instance := newPostgresTestDbInstanceCr() postgresDbCr := newPostgresTestDbCr(instance) - c := ConnectionStringFields{ + c := SecretsTemplatesFields{ DatabaseHost: "postgres", DatabasePort: 5432, UserName: testDbcred.Username, @@ -165,7 +368,7 @@ func TestPsqlDefaultConnectionStringGeneratationWithoutProxy(t *testing.T) { func TestMysqlDefaultConnectionStringGeneratationWithoutProxy(t *testing.T) { mysqlDbCr := newMysqlTestDbCr() - c := ConnectionStringFields{ + c := SecretsTemplatesFields{ DatabaseHost: "mysql", DatabasePort: 3306, UserName: testDbcred.Username, @@ -210,7 +413,7 @@ func TestPsqlCustomConnectionStringGeneratation(t *testing.T) { postfix := "<-for_storing_data_you_know" postgresDbCr.Spec.ConnectionStringTemplate = fmt.Sprintf("%s{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}%s", prefix, postfix) - c := ConnectionStringFields{ + c := SecretsTemplatesFields{ DatabaseHost: "postgres", DatabasePort: 5432, UserName: testDbcred.Username, @@ -235,7 +438,7 @@ func TestWrongTemplateConnectionStringGeneratation(t *testing.T) { postgresDbCr.Spec.ConnectionStringTemplate = "{{ .Protocol }}://{{ .User }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" _, err := generateConnectionString(postgresDbCr, testDbcred) - errSubstr := "can't evaluate field User in type controllers.ConnectionStringFields" + errSubstr := "can't evaluate field User in type controllers.SecretsTemplatesFields" assert.Contains(t, err.Error(), errSubstr, "the error doesn't contain expected substring") } diff --git a/docs/creatingdatabases.md b/docs/creatingdatabases.md index 0ccca38a..41d178d8 100644 --- a/docs/creatingdatabases.md +++ b/docs/creatingdatabases.md @@ -44,14 +44,11 @@ spec: backup: enable: false # turn it to true when you want to use back up feature. currently only support postgres cron: "0 0 * * *" - connectionStringTemplate: | - "jdbc:{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" # provide a custom template to generate a database connection string + secretsTemplates: + CONNECTION_STRING: "jdbc:{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" + PASSWORD_USER: "{{ .Password }}_{{ .User }}" ``` -If your application needs a connections string in another format, you can provide your own template via the connectionStringTemplate field. When the `connectionStringTemplate` is empty, the default template is used: -``` -{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }} -``` -These fields can be used to generate a custom template: +With `secretsTemplates` you can add fields to the database secret that are composed by any string and by any of the following templated values: ```YAML - Protocol: Depending on db engine. Possible values are mysql/postgresql - UserName: The same value as for database user in the creds secret @@ -60,6 +57,10 @@ These fields can be used to generate a custom template: - DatabasePort: The same value as for db port in the connection configmap - DatabaseName: The same value as for db host in the creds secret ``` +If no secretsTemplates are specified, the default one will be used: +```YAML +CONNECTION_STRING: "jdbc:{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}" +``` For `postgres` it's also possible to drop the `Public` schema after the database creation, or to create additional schemas. To do that, you need to provide these fields: ```YAML diff --git a/go.mod b/go.mod index 06f2dbf4..b2717e77 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( k8s.io/apiextensions-apiserver v0.23.5 k8s.io/apimachinery v0.24.0 k8s.io/client-go v0.24.0 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/controller-runtime v0.11.2 ) @@ -68,7 +69,6 @@ require ( github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.4.0 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect @@ -89,7 +89,6 @@ require ( k8s.io/component-base v0.23.5 // indirect k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index 0aeda2d5..74c7b649 100644 --- a/go.sum +++ b/go.sum @@ -579,14 +579,12 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= @@ -1184,7 +1182,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/utils/database/types.go b/pkg/utils/database/types.go index f0aa0cd3..4fac536a 100644 --- a/pkg/utils/database/types.go +++ b/pkg/utils/database/types.go @@ -21,7 +21,7 @@ type Credentials struct { Name string Username string Password string - ConnectionString string + TemplatedSecrets map[string]string } // DatabaseAddress contains host and port of a database instance