METAL-3262 proxy serviceMonitor monitoring (#124)

* METAL-3262 proxy serviceMonitor monitoring

* METAL-3262 proxy serviceMonitor monitoring

* METAL-3262 fix some comments

* METAL-3262 change chart version, it's a feature change for db-operator

* METAL-3262 change monitoring status to dbinstance

Makefile

@@ -69,14 +69,14 @@ minidashboard: ## open minikube dashboard
 miniimage: build
 	@minikube image load my-image.tar
 
-k3d_setup: k3d_install k3d_image helm
+k3d_setup: k3d_install k3d_image helm ## create a k3d cluster locally and install db-operator
 
 k3d_install:
 	@wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
 	@k3d cluster create myk3s -i rancher/k3s:$(K8S_VERSION)-k3s1
 	@kubectl get pod
 
-k3d_image: build
+k3d_image: build ## rebuild the docker images and upload into your k3d cluster
 	@k3d image import my-image.tar -c myk3s
 
 ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.

charts/db-operator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 type: application
 kubeVersion: ">= 1.19 <= 1.22"
-appVersion: "1.2.0"
+appVersion: "1.3.0"
 description: A Database Operator
 name: db-operator
-version: 1.0.0
+version: 1.1.0

charts/db-operator/templates/rbac.yaml

@@ -48,6 +48,27 @@ rules:
   verbs:
   - create
   - update
+- apiGroups:
+  - "apiextensions.k8s.io"
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+  resourceNames:
+  - servicemonitors.monitoring.coreos.com
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - create
+  - update
+  - patch
+  - get
+  - list
+  - watch
 - apiGroups:
   - extensions
   resources:

charts/db-operator/values-stanley.yaml

@@ -3,7 +3,7 @@ image:
   tag: latest
   pullPolicy: IfNotPresent
 
-resources: 
+resources:
   limits:
    cpu: "0.5"
    memory: 512Mi

controllers/database_controller.go

@@ -33,6 +33,7 @@ import (
 	"github.com/kloeckner-i/db-operator/pkg/utils/proxy"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
+	crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -501,6 +502,40 @@ func (r *DatabaseReconciler) createProxy(ctx context.Context, dbcr *kciv1alpha1.
 		}
 	}
 
+	crdList := crdv1.CustomResourceDefinitionList{}
+	err = r.List(ctx, &crdList)
+	if err != nil {
+		return err
+	}
+
+	isMonitoringEnabled, err := dbcr.IsMonitoringEnabled()
+	if err != nil {
+		return err
+	}
+
+	if isMonitoringEnabled && inCrdList(crdList, "servicemonitors.monitoring.coreos.com") {
+		// create proxy PromServiceMonitor
+		promSvcMon, err := proxy.BuildServiceMonitor(proxyInterface)
+		if err != nil {
+			return err
+		}
+		err = r.Create(ctx, promSvcMon)
+		if err != nil {
+			if k8serrors.IsAlreadyExists(err) {
+				patch := client.MergeFrom(promSvcMon)
+				err := r.Patch(ctx, promSvcMon, patch)
+				if err != nil {
+					logrus.Errorf("DB: namespace=%s, name=%s failed patching prometheus service monitor", dbcr.Namespace, dbcr.Name)
+					return err
+				}
+			} else {
+				// failed to create service
+				logrus.Errorf("DB: namespace=%s, name=%s failed creating prometehus service monitor", dbcr.Namespace, dbcr.Name)
+				return err
+			}
+		}
+	}
+
 	engine, _ := dbcr.GetEngineType()
 	dbcr.Status.ProxyStatus.ServiceName = svc.Name
 	for _, svcPort := range svc.Spec.Ports {

controllers/helper.go

@@ -21,6 +21,7 @@ import (
 
 	kciv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1"
 	"github.com/kloeckner-i/db-operator/pkg/utils/kci"
+	crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 )
 
 func isDBSpecChanged(dbcr *kciv1alpha1.Database) bool {
@@ -78,3 +79,13 @@ func containsString(slice []string, s string) bool {
 	}
 	return false
 }
+
+// inCrdList returns true if monitoring is enabled in DbInstance spec.
+func inCrdList(crds crdv1.CustomResourceDefinitionList, api string) bool {
+	for _, crd := range crds.Items {
+		if crd.Name == api {
+			return true
+		}
+	}
+	return false
+}

go.mod

@@ -11,15 +11,16 @@ require (
 	github.com/kloeckner-i/can-haz-password v0.1.0
 	github.com/lib/pq v1.10.2
 	github.com/mitchellh/hashstructure v1.1.0
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.52.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
 	google.golang.org/api v0.47.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.22.1
-	k8s.io/apimachinery v0.22.1
-	k8s.io/client-go v0.22.1
+	k8s.io/api v0.22.3
+	k8s.io/apimachinery v0.22.3
+	k8s.io/client-go v0.22.3
 	sigs.k8s.io/controller-runtime v0.10.0
 )
 
@@ -74,11 +75,11 @@ require (
 	google.golang.org/protobuf v1.26.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiextensions-apiserver v0.22.1 // indirect
-	k8s.io/component-base v0.22.1 // indirect
-	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/apiextensions-apiserver v0.22.3 // indirect
+	k8s.io/component-base v0.22.3 // indirect
+	k8s.io/klog/v2 v2.10.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 // indirect
-	k8s.io/utils v0.0.0-20210802155522-efc7438f0176 // indirect
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -399,6 +399,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.52.1 h1:sAQ0rZCj/PJOVxllCP6alH4a7P5TjHuqywBaHx7uZTo=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.52.1/go.mod h1:f7HML3SGY4Bf10YMdSWKUf2BdIIzQqlAvYh84px05BQ=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -940,30 +942,39 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY=
 k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
-k8s.io/apiextensions-apiserver v0.22.1 h1:YSJYzlFNFSfUle+yeEXX0lSQyLEoxoPJySRupepb0gE=
+k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4=
+k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs=
 k8s.io/apiextensions-apiserver v0.22.1/go.mod h1:HeGmorjtRmRLE+Q8dJu6AYRoZccvCMsghwS8XTUYb2c=
-k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM=
+k8s.io/apiextensions-apiserver v0.22.3 h1:bKku7MqawIbtTZc084BZoMV4fz0WZuvCnB5E+yrQXGM=
+k8s.io/apiextensions-apiserver v0.22.3/go.mod h1:f4plF+CXeqI89jAXL0Ml4LI/kSAZ54JS94+XOX1sae8=
 k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/apimachinery v0.22.3 h1:mrvBG5CZnEfwgpVqWcrRKvdsYECTrhAR6cApAgdsflk=
+k8s.io/apimachinery v0.22.3/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
 k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400=
-k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw=
+k8s.io/apiserver v0.22.3/go.mod h1:oam7lH/F1Kto/WTamyQYrD68fS0mGUBORAFf6x/9Mxs=
 k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
+k8s.io/client-go v0.22.3 h1:6onkOSc+YNdwq5zXE0wFXicq64rrym+mXwHu/CPVGO4=
+k8s.io/client-go v0.22.3/go.mod h1:ElDjYf8gvZsKDYexmsmnMQ0DYO8W9RwBjfQ1PI53yow=
 k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
-k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo=
+k8s.io/code-generator v0.22.3/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=
 k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo=
+k8s.io/component-base v0.22.3 h1:/+hryAW03u3FpJQww+GSMsArJNUbGjH66lrgxaRynLU=
+k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
 k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/klog/v2 v2.10.0 h1:R2HDMDJsHVTHA2n4RjwbeYXdOcBymXdX/JRb1v0VGhE=
+k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 h1:pbudjNtv90nOgR0/DUhPwKHnQ55Khz8+sNhJBIK7A5M=
 k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
 k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g=
+k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

main.go

@@ -25,6 +25,7 @@ import (
 	kcirocksv1alpha1 "github.com/kloeckner-i/db-operator/api/v1alpha1"
 	"github.com/kloeckner-i/db-operator/controllers"
 	"github.com/kloeckner-i/db-operator/pkg/config"
+	"github.com/kloeckner-i/db-operator/pkg/utils/thirdpartyapi"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -47,6 +48,8 @@ func init() {
 
 	utilruntime.Must(kcirocksv1alpha1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
+
+	thirdpartyapi.AppendToScheme(scheme)
 }
 
 func main() {

pkg/utils/proxy/cloudproxy.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 
 	"github.com/kloeckner-i/db-operator/pkg/config"
+	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	v1apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -174,3 +175,26 @@ func (cp *CloudProxy) container() (v1.Container, error) {
 func (cp *CloudProxy) buildConfigMap() (*v1.ConfigMap, error) {
 	return nil, nil
 }
+
+func (cp *CloudProxy) buildServiceMonitor() (*promv1.ServiceMonitor, error) {
+	Endpoint := promv1.Endpoint{
+		Port: "metrics",
+	}
+
+	return &promv1.ServiceMonitor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cp.NamePrefix + "-sm",
+			Namespace: cp.Namespace,
+			Labels:    cp.Labels,
+		},
+		Spec: promv1.ServiceMonitorSpec{
+			Endpoints: []promv1.Endpoint{Endpoint},
+			NamespaceSelector: promv1.NamespaceSelector{
+				MatchNames: []string{cp.Namespace},
+			},
+			Selector: metav1.LabelSelector{
+				MatchLabels: cp.Labels,
+			},
+		},
+	}, nil
+}

pkg/utils/proxy/create.go

@@ -17,6 +17,7 @@
 package proxy
 
 import (
+	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/sirupsen/logrus"
 	v1apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -54,3 +55,14 @@ func BuildConfigmap(proxy Proxy) (*v1.ConfigMap, error) {
 
 	return cm, nil
 }
+
+// BuildServiceMonitor builds kubernetes prometheus ServiceMonitor CR object used for monitoring
+func BuildServiceMonitor(proxy Proxy) (*promv1.ServiceMonitor, error) {
+	promSerMon, err := proxy.buildServiceMonitor()
+	if err != nil {
+		logrus.Error("failed building promServiceMonitor configmap")
+		return nil, err
+	}
+
+	return promSerMon, nil
+}

pkg/utils/proxy/types.go

@@ -17,6 +17,7 @@
 package proxy
 
 import (
+	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	v1apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 )
@@ -25,5 +26,6 @@ import (
 type Proxy interface {
 	buildDeployment() (*v1apps.Deployment, error)
 	buildService() (*v1.Service, error)
+	buildServiceMonitor() (*promv1.ServiceMonitor, error)
 	buildConfigMap() (*v1.ConfigMap, error)
 }

pkg/utils/thirdpartyapi/promv1.go

@@ -0,0 +1,15 @@
+package thirdpartyapi
+
+import (
+	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func AppendToScheme(scheme *runtime.Scheme) {
+	scheme.AddKnownTypes(crdv1.SchemeGroupVersion, &crdv1.CustomResourceDefinitionList{}, &crdv1.CustomResourceDefinition{})
+	metav1.AddToGroupVersion(scheme, crdv1.SchemeGroupVersion)
+	scheme.AddKnownTypes(promv1.SchemeGroupVersion, &promv1.ServiceMonitor{})
+	metav1.AddToGroupVersion(scheme, promv1.SchemeGroupVersion)
+}