METAL-3331 - Add database URL feature with template (#131)

Makefile

@@ -52,22 +52,24 @@ lint: $(SRC)
 vet: $(SRC)
 	@go vet ./...
 
-minisetup: miniup miniimage helm
+k3s_mac_lima_create: 
+	limactl start --tty=false ./resources/lima/k3s.yaml
 
-miniup: ## start minikube
-	@minikube start --kubernetes-version=$(K8S_VERSION) --cpus 2 --memory 4096
+k3s_mac_lima_start:
+	limactl start k3s
 
-minidown: ## stop minikube
-	@minikube stop
+k3s_mac_lima_helm:
+	mkdir -p "$${HOME}/.lima/k3s/conf"
+	limactl shell k3s sudo cat /etc/rancher/k3s/k3s.yaml >$${HOME}/.lima/k3s/conf/kubeconfig.yaml
+	@helm upgrade --install --namespace operator --create-namespace my-dboperator charts/db-operator -f charts/db-operator/values.yaml -f charts/db-operator/values-local.yaml --kubeconfig $${HOME}/.lima/k3s/conf/kubeconfig.yaml
+	echo "Don't forget to use k3s docker coonfig \nexport KUBECONFIG=$${HOME}/.lima/k3s/conf/kubeconfig.yaml"
 
-minidelete: ## delete minikube
-	@minikube delete
+k3s_mac_deploy: build k3s_mac_image k3s_mac_lima_helm
 
-minidashboard: ## open minikube dashboard
-	@minikube dashboard
-
-miniimage: build
-	@minikube image load my-image.tar
+k3s_mac_image:
+	limactl copy my-image.tar k3s:/tmp/db.tar
+	limactl shell k3s sudo k3s ctr images import /tmp/db.tar
+	limactl shell k3s rm -f /tmp/db.tar
 
 k3d_setup: k3d_install k3d_image helm ## create a k3d cluster locally and install db-operator
 

api/v1alpha1/database_types.go

@@ -31,6 +31,11 @@ type DatabaseSpec struct {
 	DeletionProtected bool           `json:"deletionProtected"`
 	Backup            DatabaseBackup `json:"backup"`
 	Extensions        []string       `json:"extensions,omitempty"`
+	// ConnectionStringTemplate field can be used to pass a custom template for generating a db connection string.
+	// These keywords can be used: Protocol, DatabaseHost, DatabasePort, UserName, Password, DatabaseName.
+	// Default template looks like this: 
+	// "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}"
+	ConnectionStringTemplate string `json:"connectionStringTemplate,omitempty"`
 }
 
 // DatabaseStatus defines the observed state of Database

charts/db-operator/crds/kci.rocks_databases.yaml

@@ -71,6 +71,14 @@ spec:
                 - cron
                 - enable
                 type: object
+              connectionStringTemplate:
+                description: 'ConnectionStringTemplate field can be used to pass a
+                  custom template for generating a db connection string. These keywords
+                  can be used: Protocol, DatabaseHost, DatabasePort, UserName, Password,
+                  DatabaseName. Default template looks like this: "{{ .Protocol }}://{{
+                  .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort
+                  }}/{{ .DatabaseName }}"'
+                type: string
               deletionProtected:
                 type: boolean
               extensions:

config/crd/bases/kci.rocks_databases.yaml

@@ -71,6 +71,14 @@ spec:
                 - cron
                 - enable
                 type: object
+              connectionStringTemplate:
+                description: 'ConnectionStringTemplate field can be used to pass a
+                  custom template for generating a db connection string. These keywords
+                  can be used: Protocol, DatabaseHost, DatabasePort, UserName, Password,
+                  DatabaseName. Default template looks like this: "{{ .Protocol }}://{{
+                  .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort
+                  }}/{{ .DatabaseName }}"'
+                type: string
               deletionProtected:
                 type: boolean
               extensions:

controllers/database_controller.go

@@ -59,6 +59,7 @@ var (
 	dbPhaseCreate               = "Creating"
 	dbPhaseInstanceAccessSecret = "InstanceAccessSecretCreating"
 	dbPhaseProxy                = "ProxyCreating"
+	dbPhaseConnectionString     = "ConnectionStringCreating"
 	dbPhaseConfigMap            = "InfoConfigMapCreating"
 	dbPhaseMonitoring           = "MonitoringCreating"
 	dbPhaseBackupJob            = "BackupJobCreating"
@@ -198,6 +199,12 @@ func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 			if err != nil {
 				return r.manageError(ctx, dbcr, err, true)
 			}
+			dbcr.Status.Phase = dbPhaseConnectionString
+		case dbPhaseConnectionString:
+			err := r.createConnectionString(ctx, dbcr)
+			if err != nil {
+				return r.manageError(ctx, dbcr, err, true)
+			}
 			dbcr.Status.Phase = dbPhaseConfigMap
 		case dbPhaseConfigMap:
 			err := r.createInfoConfigMap(ctx, dbcr)
@@ -282,18 +289,7 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph
 				logrus.Errorf("can not generate credentials for database - %s", err)
 				return err
 			}
-			newDatabaseSecret := &corev1.Secret{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "Secret",
-					APIVersion: "v1",
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      dbcr.Spec.SecretName,
-					Namespace: dbcr.Namespace,
-					Labels:    kci.BaseLabelBuilder(),
-				},
-				Data: secretData,
-			}
+			newDatabaseSecret := kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.Namespace, secretData)
 			err = r.Create(ctx, newDatabaseSecret)
 			if err != nil {
 				// failed to create secret
@@ -305,7 +301,6 @@ func (r *DatabaseReconciler) createDatabase(ctx context.Context, dbcr *kciv1alph
 			return err
 		}
 	}
-
 	databaseCred, err := parseDatabaseSecretData(dbcr, databaseSecret.Data)
 	if err != nil {
 		// failed to parse database credential from secret
@@ -549,6 +544,44 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
 	return nil
 }
 
+func (r *DatabaseReconciler) createConnectionString(ctx context.Context, dbcr *kciv1alpha1.Database) error {
+	// First of all the password should be taken from secret because it's not stored anywhere else
+	databaseSecret, err := r.getDatabaseSecret(ctx, dbcr)
+	if err != nil {
+		return err
+	}
+	// Then parse the secret to get the password
+	databaseCred, err := parseDatabaseSecretData(dbcr, databaseSecret.Data)
+	if err != nil {
+		return err
+	}
+	// Fill ConnectionStringFields struct
+	dbUrl := ConnectionStringFields{
+		DatabaseHost: dbcr.Status.ProxyStatus.ServiceName,
+		DatabasePort: dbcr.Status.ProxyStatus.SQLPort,
+		UserName:     dbcr.Status.UserName,
+		Password:     databaseCred.Password,
+		DatabaseName: dbcr.Status.DatabaseName,
+	}
+	// Generate the connection string
+	dbConnectionString, err := generateConnectionString(dbcr, dbUrl)
+	if err != nil {
+		return err
+	}
+	// Update database-credentials secret.
+	if databaseCred.ConnectionString == dbConnectionString {
+		return nil
+	}
+	logrus.Debugf("DB: namespace=%s, name=%s updating credentials secret", dbcr.Namespace, dbcr.Name)
+	newSecret := addConnectionStringToSecret(dbcr, databaseSecret.Data, dbConnectionString)
+	err = r.Update(ctx, newSecret, &client.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	logrus.Infof("DB: namespace=%s, name=%s connection string is added to credentials secret", dbcr.Namespace, dbcr.Name)
+	return nil
+}
+
 func (r *DatabaseReconciler) createInfoConfigMap(ctx context.Context, dbcr *kciv1alpha1.Database) error {
 	instance, err := dbcr.GetInstanceRef()
 	if err != nil {

controllers/database_helper.go

@@ -17,15 +17,28 @@
 package controllers
 
 import (
+	"bytes"
 	"errors"
 	"strconv"
+	"text/template"
 
 	kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1"
 	"github.com/kloeckner-i/db-operator/pkg/utils/database"
 	"github.com/kloeckner-i/db-operator/pkg/utils/kci"
 	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
 )
 
+//ConnectionStringFields defines default fields that can be used to generate a connection string
+type ConnectionStringFields struct {
+	Protocol     string
+	DatabaseHost string
+	DatabasePort int32
+	UserName     string
+	Password     string
+	DatabaseName string
+}
+
 func determinDatabaseType(dbcr *kciv1alpha1.Database, dbCred database.Credentials) (database.Database, error) {
 	instance, err := dbcr.GetInstanceRef()
 	if err != nil {
@@ -105,44 +118,51 @@ func parseDatabaseSecretData(dbcr *kciv1alpha1.Database, data map[string][]byte)
 		return cred, err
 	}
 
+	// Connection string can be empty
+	if connectionString, ok := data["CONNECTION_STRING"]; ok {
+		cred.ConnectionString = string(connectionString)
+	} else {
+		logrus.Info("CONNECTION_STRING key does not exist in secret data")
+	}
+
 	switch engine {
 	case "postgres":
 		if name, ok := data["POSTGRES_DB"]; ok {
 			cred.Name = string(name)
 		} else {
-			return cred, errors.New("POSTGRES_DB key does not exists in secret data")
+			return cred, errors.New("POSTGRES_DB key does not exist in secret data")
 		}
 
 		if user, ok := data["POSTGRES_USER"]; ok {
 			cred.Username = string(user)
 		} else {
-			return cred, errors.New("POSTGRES_USER key does not exists in secret data")
+			return cred, errors.New("POSTGRES_USER key does not exist in secret data")
 		}
 
 		if pass, ok := data["POSTGRES_PASSWORD"]; ok {
 			cred.Password = string(pass)
 		} else {
-			return cred, errors.New("POSTGRES_PASSWORD key does not exists in secret data")
+			return cred, errors.New("POSTGRES_PASSWORD key does not exist in secret data")
 		}
 
 		return cred, nil
 	case "mysql":
 		if name, ok := data["DB"]; ok {
 			cred.Name = string(name)
 		} else {
-			return cred, errors.New("DB key does not exists in secret data")
+			return cred, errors.New("DB key does not exist in secret data")
 		}
 
 		if user, ok := data["USER"]; ok {
 			cred.Username = string(user)
 		} else {
-			return cred, errors.New("USER key does not exists in secret data")
+			return cred, errors.New("USER key does not exist in secret data")
 		}
 
 		if pass, ok := data["PASSWORD"]; ok {
 			cred.Password = string(pass)
 		} else {
-			return cred, errors.New("PASSWORD key does not exists in secret data")
+			return cred, errors.New("PASSWORD key does not exist in secret data")
 		}
 
 		return cred, nil
@@ -163,7 +183,6 @@ func generateDatabaseSecretData(dbcr *kciv1alpha1.Database) (map[string][]byte,
 	if err != nil {
 		return nil, err
 	}
-
 	dbName := dbcr.Namespace + "-" + dbcr.Name
 	dbUser := dbcr.Namespace + "-" + dbcr.Name
 	dbPassword := kci.GeneratePass()
@@ -187,3 +206,45 @@ func generateDatabaseSecretData(dbcr *kciv1alpha1.Database) (map[string][]byte,
 		return nil, errors.New("not supported engine type")
 	}
 }
+
+func generateConnectionString(dbcr *kciv1alpha1.Database, dbData ConnectionStringFields) (connString string, err error) {
+	// The string that's going to be generated if the default template is used:
+	// "postgresql://user:password@host:port/database"
+	const defaultTemplate = "{{ .Protocol }}://{{ .UserName }}:{{ .Password }}@{{ .DatabaseHost }}:{{ .DatabasePort }}/{{ .DatabaseName }}"
+
+	// If engine is 'postgres', the protocol should be postgresql
+	if dbcr.Status.InstanceRef.Spec.Engine == "postgres" {
+		dbData.Protocol = "postgresql"
+	} else {
+		dbData.Protocol = dbcr.Status.InstanceRef.Spec.Engine
+	}
+
+	// If dbcr.Spec.ConnectionString is not specified, use the defalt template
+	var tmpl string
+	if dbcr.Spec.ConnectionStringTemplate != "" {
+		tmpl = dbcr.Spec.ConnectionStringTemplate
+	} else {
+		tmpl = defaultTemplate
+	}
+
+	t, err := template.New("connection_string").Parse(tmpl)
+	if err != nil {
+		logrus.Error(err)
+		return
+	}
+
+	var connStringBytes bytes.Buffer
+	err = t.Execute(&connStringBytes, dbData)
+	if err != nil {
+		logrus.Error(err)
+		return
+	}
+
+	connString = connStringBytes.String()
+	return
+}
+
+func addConnectionStringToSecret(dbcr *kciv1alpha1.Database, secretData map[string][]byte, connectionString string) *v1.Secret {
+	secretData["CONNECTION_STRING"] = []byte(connectionString)
+	return kci.SecretBuilder(dbcr.Spec.SecretName, dbcr.GetNamespace(), secretData)
+}