refactor: Do not delete prometheus pods after configuring ConfigMaps (#252)

There is no need to delete pods to force a reload as the
configmap-reload already takes care of triggering a config reload.
The code that deletes prometheus pods has been removed along with the
related permission in the role for prometheus-service

Signed-off-by: Paolo Chila <[email protected]>

Author: pchila

eventhandling/configureEvent.go

@@ -11,8 +11,6 @@ import (
 	"gopkg.in/yaml.v2"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 
 	"github.com/keptn-contrib/prometheus-service/utils"
 
@@ -133,72 +131,41 @@ func (eh ConfigureMonitoringEventHandler) configurePrometheusAndStoreResources(e
 		if err != nil {
 			return err
 		}
-
-		// (2.1) restart prometheus by deleting prometheus pod
-		if err := eh.deletePod(env.PrometheusLabels, env.PrometheusNamespace); err != nil {
-			return err
-		}
-
-		// (2.1) restart prometheus alert manager by deleting prometheus alert manager pod
-		if err := eh.deletePod(env.AlertManagerLabels, env.AlertManagerNamespace); err != nil {
-			return err
-		}
 	}
 
 	return nil
 }
 
-func (eh ConfigureMonitoringEventHandler) deletePod(labels string, namespace string) error {
-	eh.logger.Info("Deleting Pod with labels " + labels + "...")
-
-	prometheusHelper, err := utils.NewPrometheusHelper()
-	if err != nil {
-		return err
-	}
-
-	labelArr := strings.Split(labels, ",")
-
-	for _, label := range labelArr {
-		err = prometheusHelper.DeletePod(label, namespace)
-		if err != nil {
-			return err
-		}
-	}
-
-	eh.logger.Info("Deleting Pod successfully")
-
-	return nil
-}
-
 func (eh ConfigureMonitoringEventHandler) isPrometheusInstalled() bool {
 	eh.logger.Debug("Check if prometheus service in " + env.PrometheusNamespace + " namespace is available")
-	config, err := rest.InClusterConfig()
+	svcList, err := getPrometheusServiceFromK8s()
 	if err != nil {
-		eh.logger.Debug(fmt.Sprintf("Could not initialize kubernetes client %s", err.Error()))
+		eh.logger.Errorf("Error locating prometheus service in k8s: %v", err)
 		return false
 	}
-	api, err := kubernetes.NewForConfig(config)
 
-	if err != nil {
-		eh.logger.Debug(fmt.Sprintf("Could not initialize kubernetes client %s", err.Error()))
-		return false
+	if len(svcList.Items) > 0 {
+		eh.logger.Debug("Prometheus service in " + env.PrometheusNamespace + " namespace is available")
+		return true
 	}
 
-	svc, err := api.CoreV1().Services(env.PrometheusNamespace).List(metav1.ListOptions{
-		LabelSelector: env.PrometheusLabels,
-	})
+	return false
+}
 
+func getPrometheusServiceFromK8s() (*v1.ServiceList, error) {
+	svcList, err := utils.ListK8sServicesByLabel(env.PrometheusLabels, env.PrometheusNamespace)
 	if err != nil {
-		eh.logger.Debug(fmt.Sprintf("Prometheus service in %s namespace is not available. %s", env.PrometheusNamespace, err.Error()))
-		return false
+		return nil, fmt.Errorf("prometheus service not found: %w", err)
 	}
+	return svcList, err
+}
 
-	if len(svc.Items) > 0 {
-		eh.logger.Debug("Prometheus service in " + env.PrometheusNamespace + " namespace is available")
-		return true
+func getPrometheusAlertManagerServiceFromK8s() (*v1.ServiceList, error) {
+	svcList, err := utils.ListK8sServicesByLabel(env.AlertManagerLabels, env.AlertManagerNamespace)
+	if err != nil {
+		return nil, fmt.Errorf("prometheus alert manager service not found: %w", err)
 	}
-
-	return false
+	return svcList, err
 }
 
 func (eh ConfigureMonitoringEventHandler) configurePrometheusAlertManager() error {
@@ -223,7 +190,7 @@ func (eh ConfigureMonitoringEventHandler) updatePrometheusConfigMap(eventData ke
 		return err
 	}
 
-	api, err := getKubeClient()
+	api, err := utils.GetKubeClient()
 	if err != nil {
 		return err
 	}
@@ -339,18 +306,6 @@ func (eh ConfigureMonitoringEventHandler) updatePrometheusConfigMap(eventData ke
 	return nil
 }
 
-func getKubeClient() (*kubernetes.Clientset, error) {
-	k8sConfig, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
-	api, err := kubernetes.NewForConfig(k8sConfig)
-	if err != nil {
-		return nil, err
-	}
-	return api, nil
-}
-
 func getDefaultFilterExpression(project string, stage string, service string, filters map[string]string) string {
 	filterExpression := "job='" + service + "-" + project + "-" + stage + "-primary'"
 	if filters != nil && len(filters) > 0 {

utils/k8s.go

@@ -0,0 +1,46 @@
+package utils
+
+import (
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// GetKubeClient returns a k8s ClientSet using the in-cluster config
+func GetKubeClient() (*kubernetes.Clientset, error) {
+	k8sConfig, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	api, err := kubernetes.NewForConfig(k8sConfig)
+	if err != nil {
+		return nil, err
+	}
+	return api, nil
+}
+
+// ListK8sServicesByLabel returns a k8s service list matching the passed labelSelector in the specified namespace
+func ListK8sServicesByLabel(svcLabelSelector, namespace string) (*v1.ServiceList, error) {
+	api, err := GetKubeClient()
+
+	if err != nil {
+		return nil, fmt.Errorf("could not initialize kubernetes client %w", err)
+	}
+
+	svcList, err := api.CoreV1().Services(namespace).List(
+		metav1.ListOptions{
+			LabelSelector: svcLabelSelector,
+		},
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf(
+			"service matching LabelSelector %s in %s namespace is not available. %w", svcLabelSelector, namespace,
+			err,
+		)
+	}
+	return svcList, nil
+}

utils/prometheus.go

@@ -5,23 +5,23 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
 
 	keptncommon "github.com/keptn/go-utils/pkg/lib/keptn"
 	alertConfig "github.com/prometheus/alertmanager/config"
 	"gopkg.in/yaml.v2"
-	"io/ioutil"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"math"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
 
 	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
-	"net/http"
 )
 
 const Throughput = "throughput"
@@ -118,23 +118,6 @@ func (p *PrometheusHelper) CreateConfigMap(cm *v1.ConfigMap, namespace string) e
 	return nil
 }
 
-func (p *PrometheusHelper) DeletePod(label string, namespace string) error {
-	pod_list, err := p.KubeApi.CoreV1().Pods(namespace).List(metav1.ListOptions{
-		LabelSelector: label,
-	})
-	if err != nil {
-		return err
-	}
-
-	for _, pod := range pod_list.Items {
-		err := p.KubeApi.CoreV1().Pods(namespace).Delete(pod.Name, &metav1.DeleteOptions{})
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (p *PrometheusHelper) UpdateAMConfigMap(name string, filename string, namespace string) error {
 	getCM, err := p.GetConfigMap(name, namespace)
 	if err != nil {