Merge pull request #13 from kloeckner-i/monitoring_enabled_from_dbin_cr

Add spec.monitoring.enabled in DbInstance custom resource

Makefile

@@ -53,7 +53,7 @@ deploy:
 update: build deploy
 
 test:
-	@docker run --rm --name postgres -e POSTGRES_PASSWORD=test1234 -p 5432:5432 -d postgres:9.6-alpine
+	@docker run --rm --name postgres -e POSTGRES_PASSWORD=test1234 -p 5432:5432 -d postgres:11-alpine
 	@docker run --rm --name mysql -e MYSQL_ROOT_PASSWORD=test1234 -p 3306:3306 -d mysql:5.7
 	@sleep 2
 	@go test -tags tests ./... -v -cover

docs/creatingdatabases.md

@@ -23,6 +23,7 @@ For more details about how it works check [here](howitworks.md)
 * [Creating databases](#CreatingDatabases)
 * [Connecting to the database from pods](#ConnectingToTheDatabase)
 * [Checking database status](#CheckingDatabaseStatus)
+* [Enabling postgreSQL extensions](#PostgreSQL)
 
 ### CreatingDatabases
 
@@ -176,4 +177,29 @@ Possible phases and meanings
 | `BackupJobCreating`   | Creating backup `Cronjob` when backup is enabled in the `spec` |
 | `Finishing`           | Setting status of `Database` to true |
 | `Ready`               | `Database` is created and all the configs are applied. Healthy status. |
-| `Deleting`            | `Database` is being deleted. |
+| `Deleting`            | `Database` is being deleted. |
+
+### PostgreSQL
+
+PostgreSQL extensions listed under `spec.extensions` will be enabled by DB Operator.
+DB Operator execute `CREATE EXTENSION IF NOT EXISTS` on the target database.
+
+```YAML
+apiVersion: "kci.rocks/v1alpha1"
+kind: "Database"
+metadata:
+  name: "example-db"
+spec:
+  secretName: example-db-credentials
+  instance: example-gsql
+  deletionProtected: false
+  extensions:
+    - pgcrypto
+    - uuid-ossp
+    - plpgsql
+```
+When monitoring is enabled on DbInstance spec, `pg_stat_statements` extension will be enabled.
+If below error occurs during database creation, the module must be loaded by adding pg_stat_statements to shared_preload_libraries in postgresql.conf on the server side.
+```
+ERROR: pg_stat_statements must be loaded via shared_preload_libraries
+```

helm/db-instances/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: Database Instances for db operator
 name: db-instances
-version: 0.1.0
+version: 0.2.0

helm/db-instances/templates/dbinstance.yaml

@@ -26,6 +26,8 @@ spec:
   {{- if $value.backup }}
   backup:
 {{ toYaml $value.backup | indent 4 }}
+  monitoring:
+    enabled: {{ $value.monitoring.enabled }}
   {{- end }}
   {{- if $value.google }}
   google:

helm/db-instances/templates/postgres_exporter.yaml

@@ -7,6 +7,7 @@
 {{- if .Values.dbinstances }}
 {{- range $name, $value := .Values.dbinstances }}
 {{- if eq $value.engine "postgres" }}
+{{- if $value.monitoring.enabled }}
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -74,4 +75,5 @@ spec:
         name: queries
 {{- end }}
 {{- end }}
+{{- end }}
 {{- end }}

helm/index.yaml

@@ -3,7 +3,16 @@ entries:
   db-instances:
   - apiVersion: v1
     appVersion: "1.0"
-    created: "2020-03-20T15:16:44.935374+01:00"
+    created: "2020-03-27T15:28:11.819723+01:00"
+    description: Database Instances for db operator
+    digest: 62ed091aafea816e8529790b40c6c6051ddc83b89b0e8d830d9038afbc436bf2
+    name: db-instances
+    urls:
+    - https://kloeckner-i.github.io/db-operator/helm/db-instances-0.2.0.tgz
+    version: 0.2.0
+  - apiVersion: v1
+    appVersion: "1.0"
+    created: "2020-03-27T15:28:11.819207+01:00"
     description: Database Instances for db operator
     digest: 0a9f5c620c9ee8d8e33ea96f797be989efab296fbeeca24e26c84fdb2136d704
     name: db-instances
@@ -13,7 +22,7 @@ entries:
   db-operator:
   - apiVersion: v1
     appVersion: "1.0"
-    created: "2020-03-20T15:16:44.937122+01:00"
+    created: "2020-03-27T15:28:11.821852+01:00"
     description: A Database Operator
     digest: d2d2a7f6b7913585283d72cea95770067e6c9625479e349c2e18b7dc3fbd3a52
     name: db-operator
@@ -22,11 +31,11 @@ entries:
     version: 0.1.1
   - apiVersion: v1
     appVersion: "1.0"
-    created: "2020-03-20T15:16:44.936196+01:00"
+    created: "2020-03-27T15:28:11.820584+01:00"
     description: A Database Operator
     digest: 6f41f18fe10b74edaec021defdcca99e2c65b5ed60b3c53b3d7ee7b124ccfeb0
     name: db-operator
     urls:
     - https://kloeckner-i.github.io/db-operator/helm/db-operator-0.1.0.tgz
     version: 0.1.0
-generated: "2020-03-20T15:16:44.933707+01:00"
+generated: "2020-03-27T15:28:11.818096+01:00"

integration/postgres/values.yaml

@@ -2,4 +2,4 @@ postgres:
   serviceName: pg-test-db
   adminUser: postgres
   adminPassword: test1234
-  image: postgres:11
+  image: postgres:11-alpine

pkg/apis/kci/v1alpha1/database_types.go

@@ -92,3 +92,13 @@ func (db *Database) GetBackendType() (string, error) {
 
 	return instance.GetBackendType()
 }
+
+// IsMonitoringEnabled returns true if monitoring is enabled in DbInstance spec.
+func (db *Database) IsMonitoringEnabled() (bool, error) {
+	instance, err := db.GetInstanceRef()
+	if err != nil {
+		return false, err
+	}
+
+	return instance.IsMonitoringEnabled(), nil
+}

pkg/apis/kci/v1alpha1/dbinstance_types.go

@@ -16,6 +16,7 @@ type DbInstanceSpec struct {
 	Engine          string               `json:"engine"`
 	AdminUserSecret types.NamespacedName `json:"adminSecretRef"`
 	Backup          DbInstanceBackup     `json:"backup"`
+	Monitoring      DbInstanceMonitoring `json:"monitoring"`
 	Google          *GoogleInstance      `json:"google,omitempty"`
 	Generic         *GenericInstance     `json:"generic,omitempty"`
 }
@@ -51,6 +52,11 @@ type DbInstanceBackup struct {
 	Bucket string `json:"bucket"`
 }
 
+// DbInstanceMonitoring defines if exporter
+type DbInstanceMonitoring struct {
+	Enabled bool `json:"enabled"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // DbInstance is the Schema for the dbinstances API
@@ -119,3 +125,12 @@ func (dbin *DbInstance) GetBackendType() (string, error) {
 
 	return "", errors.New("no backend type defined")
 }
+
+// IsMonitoringEnabled returns boolean value if monitoring is enabled for the instance
+func (dbin *DbInstance) IsMonitoringEnabled() bool {
+	if dbin.Spec.Monitoring.Enabled == false {
+		return false
+	}
+
+	return true
+}

pkg/controller/database/databaseHelper.go

@@ -2,10 +2,11 @@ package database
 
 import (
 	"errors"
+	"strconv"
+
 	kciv1alpha1 "github.com/kloeckner-i/db-operator/pkg/apis/kci/v1alpha1"
 	database "github.com/kloeckner-i/db-operator/pkg/utils/database"
 	"github.com/kloeckner-i/db-operator/pkg/utils/kci"
-	"strconv"
 
 	"github.com/sirupsen/logrus"
 )
@@ -36,16 +37,27 @@ func determinDatabaseType(dbcr *kciv1alpha1.Database, dbCred database.Credential
 		return nil, err
 	}
 
+	monitoringEnabled, err := dbcr.IsMonitoringEnabled()
+	if err != nil {
+		logrus.Errorf("could not check if monitoring is enabled %s - %s", dbcr.Name, err)
+		return nil, err
+	}
+
 	switch engine {
 	case "postgres":
+		extList := dbcr.Spec.Extensions
+		if monitoringEnabled {
+			extList = append(dbcr.Spec.Extensions, "pg_stat_statements")
+		}
+
 		db := database.Postgres{
 			Backend:    backend,
 			Host:       host,
 			Port:       int32(port),
 			Database:   dbCred.Name,
 			User:       dbCred.Username,
 			Password:   dbCred.Password,
-			Extensions: dbcr.Spec.Extensions,
+			Extensions: extList,
 		}
 
 		return db, nil

pkg/controller/database/databaseHelper_test.go

@@ -1,32 +1,33 @@
 package database
 
 import (
-	database "github.com/kloeckner-i/db-operator/pkg/utils/database"
 	"testing"
 
+	database "github.com/kloeckner-i/db-operator/pkg/utils/database"
+
 	"github.com/stretchr/testify/assert"
 )
 
+var testDbcred = database.Credentials{Name: "testdb", Username: "testuser", Password: "password"}
+
 func TestDeterminPostgresType(t *testing.T) {
-	postgresDbCr := newPostgresTestDbCr()
-	dbcred := database.Credentials{Name: "testdb", Username: "testuser", Password: "password"}
+	postgresDbCr := newPostgresTestDbCr(newPostgresTestDbInstanceCr())
 
-	db, _ := determinDatabaseType(postgresDbCr, dbcred)
+	db, _ := determinDatabaseType(postgresDbCr, testDbcred)
 	_, ok := db.(database.Postgres)
 	assert.Equal(t, ok, true, "expected true")
 }
 
 func TestDeterminMysqlType(t *testing.T) {
 	mysqlDbCr := newMysqlTestDbCr()
-	dbcred := database.Credentials{Name: "testdb", Username: "testuser", Password: "password"}
 
-	db, _ := determinDatabaseType(mysqlDbCr, dbcred)
+	db, _ := determinDatabaseType(mysqlDbCr, testDbcred)
 	_, ok := db.(database.Mysql)
 	assert.Equal(t, ok, true, "expected true")
 }
 
 func TestParsePostgresSecretData(t *testing.T) {
-	postgresDbCr := newPostgresTestDbCr()
+	postgresDbCr := newPostgresTestDbCr(newPostgresTestDbInstanceCr())
 
 	invalidData := make(map[string][]byte)
 	invalidData["DB"] = []byte("testdb")
@@ -68,7 +69,7 @@ func TestParseMysqlSecretData(t *testing.T) {
 }
 
 func TestParseAdminSecretData(t *testing.T) {
-	postgresDbCr := newPostgresTestDbCr()
+	postgresDbCr := newPostgresTestDbCr(newPostgresTestDbInstanceCr())
 	invalidData := make(map[string][]byte)
 	invalidData["unknownkey"] = []byte("wrong")
 
@@ -84,3 +85,39 @@ func TestParseAdminSecretData(t *testing.T) {
 	assert.Equal(t, string(validData["user"]), cred.Username, "expect same values")
 	assert.Equal(t, string(validData["password"]), cred.Password, "expect same values")
 }
+
+func TestMonitoringNotEnabled(t *testing.T) {
+	instance := newPostgresTestDbInstanceCr()
+	instance.Spec.Monitoring.Enabled = false
+	postgresDbCr := newPostgresTestDbCr(instance)
+	db, _ := determinDatabaseType(postgresDbCr, testDbcred)
+	postgresInterface, _ := db.(database.Postgres)
+
+	found := false
+	for _, ext := range postgresInterface.Extensions {
+		if ext == "pg_stat_statements" {
+			found = true
+			break
+		}
+	}
+	assert.Equal(t, found, false, "expected pg_stat_statement is not included in extension list")
+}
+
+func TestMonitoringEnabled(t *testing.T) {
+	instance := newPostgresTestDbInstanceCr()
+	instance.Spec.Monitoring.Enabled = false
+	postgresDbCr := newPostgresTestDbCr(instance)
+	postgresDbCr.Status.InstanceRef.Spec.Monitoring.Enabled = true
+
+	db, _ := determinDatabaseType(postgresDbCr, testDbcred)
+	postgresInterface, _ := db.(database.Postgres)
+
+	found := false
+	for _, ext := range postgresInterface.Extensions {
+		if ext == "pg_stat_statements" {
+			found = true
+			break
+		}
+	}
+	assert.Equal(t, found, true, "expected pg_stat_statement is included in extension list")
+}