From 908b4d5234438a5618cfae4bfe6a4a137a0089de Mon Sep 17 00:00:00 2001 From: Christian Kreuzberger Date: Mon, 14 Feb 2022 09:15:34 +0100 Subject: [PATCH] refactor: use built-in send event functionality of keptn/go-utils (#267) * refactor: use built-in send event functionality of keptn/go-utils Signed-off-by: Christian Kreuzberger * refactor: use built-in send event functionality for configure-monitoring of keptn/go-utils Signed-off-by: Christian Kreuzberger * refactor: use built-in structs for alert events Signed-off-by: Christian Kreuzberger --- chart/templates/deployment.yaml | 2 +- eventhandling/alertEvent.go | 37 +++------ eventhandling/configureEvent.go | 40 +++------ eventhandling/getSliEvent.go | 139 ++++++++++---------------------- eventhandling/handler.go | 5 +- main.go | 6 ++ 6 files changed, 73 insertions(+), 156 deletions(-) diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 1908e12..88e20cc 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -104,7 +104,7 @@ spec: - name: PUBSUB_URL value: 'nats://keptn-nats-cluster' - name: PUBSUB_TOPIC - value: 'sh.keptn.event.monitoring.configure,sh.keptn.event.get-sli.triggered' + value: 'sh.keptn.event.monitoring.configure,sh.keptn.event.configure-monitoring.triggered,sh.keptn.event.get-sli.triggered' - name: PUBSUB_RECIPIENT value: '127.0.0.1' - name: STAGE_FILTER diff --git a/eventhandling/alertEvent.go b/eventhandling/alertEvent.go index 95c723a..36d11fa 100644 --- a/eventhandling/alertEvent.go +++ b/eventhandling/alertEvent.go @@ -9,6 +9,7 @@ import ( "strings" "time" + keptncommons "github.com/keptn/go-utils/pkg/lib" "github.com/keptn/go-utils/pkg/lib/keptn" keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0" @@ -24,6 +25,7 @@ type alertManagerEvent struct { Alerts []alert `json:"alerts""` } +// alert coming from prometheus type alert struct { Status string `json:"status"` Labels labels `json:"labels"` @@ -49,27 +51,10 @@ type annotations struct { Description string `json:"descriptions,omitempty"` } -type eventData struct { - Project string `json:"project,omitempty"` - Stage string `json:"stage,omitempty"` - Service string `json:"service,omitempty"` - Labels map[string]string `json:"labels"` - Problem problemData `json:"problem"` -} - -type problemData struct { - State string `json:"State,omitempty"` - ProblemID string `json:"ProblemID"` - ProblemTitle string `json:"ProblemTitle"` - ProblemDetails json.RawMessage `json:"ProblemDetails"` - PID string `json:"PID"` - ProblemURL string `json:"ProblemURL,omitempty"` - ImpactedEntity string `json:"ImpactedEntity,omitempty"` -} - // ProcessAndForwardAlertEvent reads the payload from the request and sends a valid Cloud event to the keptn event broker func ProcessAndForwardAlertEvent(rw http.ResponseWriter, requestBody []byte, logger *keptn.Logger, shkeptncontext string) { var event alertManagerEvent + logger.Info("Received alert from Prometheus Alertmanager:" + string(requestBody)) err := json.Unmarshal(requestBody, &event) if err != nil { @@ -85,20 +70,16 @@ func ProcessAndForwardAlertEvent(rw http.ResponseWriter, requestBody []byte, log return } - newProblemData := problemData{ + newEventData := keptncommons.ProblemEventData{ State: problemState, ProblemID: "", ProblemTitle: event.Alerts[0].Annotations.Summary, ProblemDetails: json.RawMessage(`{"problemDetails":"` + event.Alerts[0].Annotations.Description + `"}`), ProblemURL: event.Alerts[0].GeneratorURL, ImpactedEntity: event.Alerts[0].Labels.PodName, - } - - newEventData := eventData{ - Project: event.Alerts[0].Labels.Project, - Stage: event.Alerts[0].Labels.Stage, - Service: event.Alerts[0].Labels.Service, - Problem: newProblemData, + Project: event.Alerts[0].Labels.Project, + Stage: event.Alerts[0].Labels.Stage, + Service: event.Alerts[0].Labels.Service, } if event.Alerts[0].Fingerprint != "" { @@ -120,7 +101,8 @@ func ProcessAndForwardAlertEvent(rw http.ResponseWriter, requestBody []byte, log } } -func createAndSendCE(problemData eventData, shkeptncontext string) error { +// createAndSendCE create a new problem.triggered event and send it to Keptn +func createAndSendCE(problemData keptncommons.ProblemEventData, shkeptncontext string) error { source, _ := url.Parse("prometheus") eventType := keptnv2.GetTriggeredEventType(problemData.Stage + "." + remediationTaskName) @@ -146,6 +128,7 @@ func createAndSendCE(problemData eventData, shkeptncontext string) error { return nil } +// createOrApplyKeptnContext re-uses the existing Keptn Context or creates a new one based on prometheus fingerprint func createOrApplyKeptnContext(contextID string) string { uuid.SetRand(nil) keptnContext := uuid.New().String() diff --git a/eventhandling/configureEvent.go b/eventhandling/configureEvent.go index 4c9c05e..7cab04d 100644 --- a/eventhandling/configureEvent.go +++ b/eventhandling/configureEvent.go @@ -70,10 +70,6 @@ type alertingAnnotations struct { // HandleEvent processes an event func (eh ConfigureMonitoringEventHandler) HandleEvent() error { - - var shkeptncontext string - _ = eh.event.Context.ExtensionAs("shkeptncontext", &shkeptncontext) - eventData := &keptnevents.ConfigureMonitoringEventData{} if err := eh.event.DataAs(eventData); err != nil { return err @@ -85,10 +81,10 @@ func (eh ConfigureMonitoringEventHandler) HandleEvent() error { err := eh.configurePrometheusAndStoreResources(eventData) if err != nil { eh.logger.Error(err.Error()) - return eh.handleError(eventData, err.Error()) + return eh.handleError(err.Error()) } - if err = eh.sendConfigureMonitoringFinishedEvent(eventData, keptnv2.StatusSucceeded, keptnv2.ResultPass, "Prometheus successfully configured and rule created"); err != nil { + if err = eh.sendConfigureMonitoringFinishedEvent(keptnv2.StatusSucceeded, keptnv2.ResultPass, "Prometheus successfully configured and rule created"); err != nil { eh.logger.Error(err.Error()) } return nil @@ -433,37 +429,23 @@ func retrieveSLOs(eventData keptnevents.ConfigureMonitoringEventData, stage stri return &slos, nil } -func (eh ConfigureMonitoringEventHandler) sendConfigureMonitoringFinishedEvent(configureMonitoringData *keptnevents.ConfigureMonitoringEventData, status keptnv2.StatusType, result keptnv2.ResultType, msg string) error { - cmFinishedEvent := &keptnv2.ConfigureMonitoringFinishedEventData{ - EventData: keptnv2.EventData{ - Project: configureMonitoringData.Project, - Service: configureMonitoringData.Service, - Status: status, - Result: result, - Message: msg, - }, - } - keptnContext, _ := eh.event.Context.GetExtension("shkeptncontext") - triggeredID := eh.event.Context.GetID() - - event := cloudevents.NewEvent() - event.SetSource(utils.ServiceName) - event.SetDataContentType(cloudevents.ApplicationJSON) - event.SetType(keptnv2.GetFinishedEventType(keptnv2.ConfigureMonitoringTaskName)) - event.SetData(cloudevents.ApplicationJSON, cmFinishedEvent) - event.SetExtension("shkeptncontext", keptnContext) - event.SetExtension("triggeredid", triggeredID) +func (eh ConfigureMonitoringEventHandler) sendConfigureMonitoringFinishedEvent(status keptnv2.StatusType, result keptnv2.ResultType, msg string) error { + _, err := eh.keptnHandler.SendTaskFinishedEvent(&keptnv2.EventData{ + Status: status, + Result: result, + Message: msg, + }, utils.ServiceName) - if err := eh.keptnHandler.SendCloudEvent(event); err != nil { + if err != nil { return fmt.Errorf("could not send %s event: %s", keptnv2.GetFinishedEventType(keptnv2.ConfigureMonitoringTaskName), err.Error()) } return nil } -func (eh ConfigureMonitoringEventHandler) handleError(e *keptnevents.ConfigureMonitoringEventData, msg string) error { +func (eh ConfigureMonitoringEventHandler) handleError(msg string) error { //logger.Error(msg) - if err := eh.sendConfigureMonitoringFinishedEvent(e, keptnv2.StatusErrored, keptnv2.ResultFailed, msg); err != nil { + if err := eh.sendConfigureMonitoringFinishedEvent(keptnv2.StatusErrored, keptnv2.ResultFailed, msg); err != nil { // an additional error occurred when trying to send configure monitoring finished back to Keptn eh.logger.Error(err.Error()) } diff --git a/eventhandling/getSliEvent.go b/eventhandling/getSliEvent.go index 52dc9b9..cff1ef5 100644 --- a/eventhandling/getSliEvent.go +++ b/eventhandling/getSliEvent.go @@ -10,9 +10,7 @@ import ( "strings" cloudevents "github.com/cloudevents/sdk-go/v2" - "github.com/cloudevents/sdk-go/v2/types" "github.com/keptn-contrib/prometheus-service/utils" - keptncommon "github.com/keptn/go-utils/pkg/lib/keptn" keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -45,31 +43,56 @@ func (eh GetSliEventHandler) HandleEvent() error { return nil } - // get shkeptncontext - keptnCtx, err := types.ToString(eh.event.Context.GetExtensions()["shkeptncontext"]) + // send started event + _, err = eh.keptnHandler.SendTaskStartedEvent(eventData, utils.ServiceName) + if err != nil { - return fmt.Errorf("could not determine keptnContext of input event: %s", err.Error()) + errMsg := fmt.Sprintf("Failed to send task started CloudEvent (%s), aborting...", err.Error()) + log.Println(errMsg) + return err } - // create empty SLI Results Array - var sliResults = []*keptnv2.SLIResult{} - - // 1: send .started event, indicating that we accepted it - if err = sendGetSLIStartedEvent(eh.event, eventData, keptnCtx); err != nil { - return sendGetSLIFinishedEvent(eh.event, eventData, sliResults, err, keptnCtx) - } + // create SLI Results + var sliResults []*keptnv2.SLIResult // 2: try to fetch metrics into sliResults - if sliResults, err = retrieveMetrics(eh.event, eventData); err != nil { + if sliResults, err = retrieveMetrics(eventData, eh.keptnHandler); err != nil { // failed to fetch metrics, send a finished event with the error - return sendGetSLIFinishedEvent(eh.event, eventData, sliResults, err, keptnCtx) + _, err = eh.keptnHandler.SendTaskFinishedEvent(&keptnv2.EventData{ + Status: keptnv2.StatusErrored, + Result: keptnv2.ResultFailed, + Message: err.Error(), + }, utils.ServiceName) + + return err } - // 3: success; send .finished event with metrics (sliResults) - return sendGetSLIFinishedEvent(eh.event, eventData, sliResults, nil, keptnCtx) + // construct finished event data + getSliFinishedEventData := &keptnv2.GetSLIFinishedEventData{ + EventData: keptnv2.EventData{ + Status: keptnv2.StatusSucceeded, + Result: keptnv2.ResultPass, + }, + GetSLI: keptnv2.GetSLIFinished{ + IndicatorValues: sliResults, + Start: eventData.GetSLI.Start, + End: eventData.GetSLI.End, + }, + } + + // send get-sli.finished event with SLI DAta + _, err = eh.keptnHandler.SendTaskFinishedEvent(getSliFinishedEventData, utils.ServiceName) + + if err != nil { + errMsg := fmt.Sprintf("Failed to send task finished CloudEvent (%s), aborting...", err.Error()) + log.Println(errMsg) + return err + } + + return nil } -func retrieveMetrics(event cloudevents.Event, eventData *keptnv2.GetSLITriggeredEventData) ([]*keptnv2.SLIResult, error) { +func retrieveMetrics(eventData *keptnv2.GetSLITriggeredEventData, keptnHandler *keptnv2.Keptn) ([]*keptnv2.SLIResult, error) { log.Printf("Retrieving Prometheus metrics") clusterConfig, err := rest.InClusterConfig() @@ -84,16 +107,12 @@ func retrieveMetrics(event cloudevents.Event, eventData *keptnv2.GetSLITriggered return nil, errors.New("could not create Kubernetes client") } + // get prometheus API URL for the provided Project from Kubernetes Config Map prometheusAPIURL, err := getPrometheusAPIURL(eventData.Project, kubeClient.CoreV1()) if err != nil { return nil, err } - keptnHandler, err := keptnv2.NewKeptn(&event, keptncommon.KeptnOpts{}) - if err != nil { - return nil, err - } - // Create a new Prometheus Handler prometheusHandler := utils.NewPrometheusHandler( prometheusAPIURL, @@ -103,6 +122,7 @@ func retrieveMetrics(event cloudevents.Event, eventData *keptnv2.GetSLITriggered eventData.GetSLI.CustomFilters, ) + // get SLI queries (from SLI.yaml) projectCustomQueries, err := getCustomQueries(keptnHandler, eventData.Project, eventData.Stage, eventData.Service) if err != nil { log.Println("retrieveMetrics: Failed to get custom queries for project " + eventData.Project) @@ -199,78 +219,3 @@ func generatePrometheusURL(pc *prometheusCredentials) string { } return strings.Replace(prometheusURL, " ", "", -1) } - -func sendGetSLIStartedEvent(inputEvent cloudevents.Event, eventData *keptnv2.GetSLITriggeredEventData, keptnContext interface{}) error { - - source, _ := url.Parse(utils.ServiceName) - - getSLIStartedEvent := keptnv2.GetSLIStartedEventData{ - EventData: keptnv2.EventData{ - Project: eventData.Project, - Stage: eventData.Stage, - Service: eventData.Service, - Labels: eventData.Labels, - Status: keptnv2.StatusSucceeded, - Result: keptnv2.ResultPass, - }, - } - - event := cloudevents.NewEvent() - event.SetType(keptnv2.GetStartedEventType(keptnv2.GetSLITaskName)) - event.SetSource(source.String()) - event.SetDataContentType(cloudevents.ApplicationJSON) - event.SetExtension("shkeptncontext", keptnContext) - event.SetExtension("triggeredid", inputEvent.ID()) - event.SetData(cloudevents.ApplicationJSON, getSLIStartedEvent) - - return sendEvent(event) -} - -func sendGetSLIFinishedEvent(inputEvent cloudevents.Event, eventData *keptnv2.GetSLITriggeredEventData, indicatorValues []*keptnv2.SLIResult, err error, keptnContext interface{}) error { - source, _ := url.Parse(utils.ServiceName) - var status = keptnv2.StatusSucceeded - var result = keptnv2.ResultPass - var message = "" - - if err != nil { - status = keptnv2.StatusErrored - result = keptnv2.ResultFailed - message = err.Error() - } - - getSLIEvent := keptnv2.GetSLIFinishedEventData{ - EventData: keptnv2.EventData{ - Project: eventData.Project, - Stage: eventData.Stage, - Service: eventData.Service, - Labels: eventData.Labels, - Status: status, - Result: result, - Message: message, - }, - GetSLI: keptnv2.GetSLIFinished{ - IndicatorValues: indicatorValues, - Start: eventData.GetSLI.Start, - End: eventData.GetSLI.End, - }, - } - - event := cloudevents.NewEvent() - event.SetType(keptnv2.GetFinishedEventType(keptnv2.GetSLITaskName)) - event.SetSource(source.String()) - event.SetDataContentType(cloudevents.ApplicationJSON) - event.SetExtension("shkeptncontext", keptnContext) - event.SetExtension("triggeredid", inputEvent.ID()) - event.SetData(cloudevents.ApplicationJSON, getSLIEvent) - - return sendEvent(event) -} - -func sendEvent(event cloudevents.Event) error { - keptnHandler, err := keptnv2.NewKeptn(&event, keptncommon.KeptnOpts{}) - if err != nil { - return err - } - - return keptnHandler.SendCloudEvent(event) -} diff --git a/eventhandling/handler.go b/eventhandling/handler.go index faf3d86..ad81d8a 100644 --- a/eventhandling/handler.go +++ b/eventhandling/handler.go @@ -4,7 +4,6 @@ import ( cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/kelseyhightower/envconfig" "github.com/keptn-contrib/prometheus-service/utils" - keptnevents "github.com/keptn/go-utils/pkg/lib" "github.com/keptn/go-utils/pkg/lib/keptn" keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0" ) @@ -33,7 +32,7 @@ func NewEventHandler(event cloudevents.Event, logger *keptn.Logger, keptnHandler logger.Error("Failed to process env var: " + err.Error()) } - if event.Type() == keptnevents.ConfigureMonitoringEventType { + if event.Type() == keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName) { return &ConfigureMonitoringEventHandler{ logger: logger, event: event, @@ -46,6 +45,8 @@ func NewEventHandler(event cloudevents.Event, logger *keptn.Logger, keptnHandler } } + logger.Error("Unknown event type " + event.Type()) + return &NoOpEventHandler{} } diff --git a/main.go b/main.go index c7d2296..02b1082 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/keptn-contrib/prometheus-service/eventhandling" "github.com/keptn-contrib/prometheus-service/utils" + keptn "github.com/keptn/go-utils/pkg/lib" "io/ioutil" "log" "net/http" @@ -79,6 +80,11 @@ func gotEvent(event cloudevents.Event) error { var shkeptncontext string _ = event.Context.ExtensionAs("shkeptncontext", &shkeptncontext) + // convert v0.1.4 spec monitoring.configure CloudEvent into a v0.2.0 spec configure-monitoring.triggered CloudEvent + if event.Type() == keptn.ConfigureMonitoringEventType { + event.SetType(keptnv2.GetTriggeredEventType(keptnv2.ConfigureMonitoringTaskName)) + } + keptnHandler, err := keptnv2.NewKeptn(&event, keptncommon.KeptnOpts{}) if err != nil {