From 0c522ae6982a0900c0f9aea67c007252d9758416 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 10:04:25 +1300 Subject: [PATCH 1/6] Constraining incoming CM types --- controllers/configmap_controller.go | 79 ++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index 60da1a1..7505368 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -37,6 +37,7 @@ import ( const InsightsLabel = "lagoon.sh/insightsType" const InsightsUpdatedAnnotationLabel = "lagoon.sh/insightsProcessed" const InsightsWriteDeferred = "lagoon.sh/insightsWriteDeferred" +const InsightsCMErrorLabel = "insights.lagoon.sh/error" type LagoonInsightsMessage struct { Payload map[string]string `json:"payload"` @@ -45,6 +46,7 @@ type LagoonInsightsMessage struct { Labels map[string]string `json:"labels"` Environment string `json:"environment"` Project string `json:"project"` + Type string `json:"type"` } // ConfigMapReconciler reconciles a ConfigMap object @@ -93,40 +95,70 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( projectName = labels["lagoon.sh/project"] } - var sendData = LagoonInsightsMessage{ - Payload: configMap.Data, - BinaryPayload: configMap.BinaryData, - Annotations: configMap.Annotations, - Labels: configMap.Labels, - Environment: environmentName, - Project: projectName, + // insightsType is a way for us to classify incoming insights data, passing + insightsType := "unclassified" + if _, ok := labels["insights.lagoon.sh/type"]; ok { + insightsType = labels["insights.lagoon.sh/type"] + } else { + // insightsType can be determined by the incoming data + if _, ok := labels["lagoon.sh/insightsType"]; ok { + switch labels["lagoon.sh/insightsType"] { + case ("sbom-gz"): + insightsType = "sbom" + case ("image-gz"): + insightsType = "inspect" + } + } } - marshalledData, err := json.Marshal(sendData) - if err != nil { - log.Error(err, "Unable to marshall config data") - return ctrl.Result{}, err - } + // We reject any type that isn't either sbom or inspect to restrict outgoing types + if insightsType != "sbom" && insightsType != "inspect" { + //// we mark this configMap as bad, and log an error + log.Error(nil, fmt.Sprintf("insightsType '%v' unrecognized - rejecting configMap", insightsType)) + err := cmlib.LabelCM(ctx, r.Client, configMap, InsightsCMErrorLabel, "invalid-type") + if err != nil { + log.Error(err, "Unable to update configmap") + return ctrl.Result{}, err + } + } else { + // Here we attempt to process types using the new name structure + + var sendData = LagoonInsightsMessage{ + Payload: configMap.Data, + BinaryPayload: configMap.BinaryData, + Annotations: configMap.Annotations, + Labels: configMap.Labels, + Environment: environmentName, + Project: projectName, + Type: insightsType, + } - err = r.MessageQWriter(marshalledData) + marshalledData, err := json.Marshal(sendData) + if err != nil { + log.Error(err, "Unable to marshall config data") + return ctrl.Result{}, err + } + err = r.MessageQWriter(marshalledData) - if err != nil { - log.Error(err, "Unable to write to message broker") + if err != nil { + log.Error(err, "Unable to write to message broker") - //In this case what we want to do is defer the processing to a couple minutes from now - future := time.Minute * 5 - futureTime := time.Now().Add(future).Unix() - err = cmlib.LabelCM(ctx, r.Client, configMap, InsightsWriteDeferred, strconv.FormatInt(futureTime, 10)) + //In this case what we want to do is defer the processing to a couple minutes from now + future := time.Minute * 5 + futureTime := time.Now().Add(future).Unix() + err = cmlib.LabelCM(ctx, r.Client, configMap, InsightsWriteDeferred, strconv.FormatInt(futureTime, 10)) + + if err != nil { + log.Error(err, "Unable to update configmap") + return ctrl.Result{}, err + } - if err != nil { - log.Error(err, "Unable to update configmap") return ctrl.Result{}, err } - return ctrl.Result{}, err } - err = cmlib.AnnotateCM(ctx, r.Client, configMap, InsightsUpdatedAnnotationLabel, time.Now().UTC().Format(time.RFC3339)) + err := cmlib.AnnotateCM(ctx, r.Client, configMap, InsightsUpdatedAnnotationLabel, time.Now().UTC().Format(time.RFC3339)) if err != nil { log.Error(err, "Unable to update configmap") @@ -151,6 +183,7 @@ func insightLabelsOnlyPredicate() predicate.Predicate { UpdateFunc: func(event event.UpdateEvent) bool { if labelExists(InsightsLabel, event.ObjectNew) && !labelExists(InsightsWriteDeferred, event.ObjectNew) && + !labelExists(InsightsCMErrorLabel, event.ObjectNew) && // We don't want to respond to errored out CMs !insightsProcessedAnnotationExists(event.ObjectNew) { return true } From 962e7f95886918959406e891392b8af5b56f8714 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 10:24:03 +1300 Subject: [PATCH 2/6] Removes extraneous processing --- controllers/configmap_controller.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index 7505368..4fedbde 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -37,7 +37,6 @@ import ( const InsightsLabel = "lagoon.sh/insightsType" const InsightsUpdatedAnnotationLabel = "lagoon.sh/insightsProcessed" const InsightsWriteDeferred = "lagoon.sh/insightsWriteDeferred" -const InsightsCMErrorLabel = "insights.lagoon.sh/error" type LagoonInsightsMessage struct { Payload map[string]string `json:"payload"` @@ -115,11 +114,6 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if insightsType != "sbom" && insightsType != "inspect" { //// we mark this configMap as bad, and log an error log.Error(nil, fmt.Sprintf("insightsType '%v' unrecognized - rejecting configMap", insightsType)) - err := cmlib.LabelCM(ctx, r.Client, configMap, InsightsCMErrorLabel, "invalid-type") - if err != nil { - log.Error(err, "Unable to update configmap") - return ctrl.Result{}, err - } } else { // Here we attempt to process types using the new name structure @@ -183,7 +177,6 @@ func insightLabelsOnlyPredicate() predicate.Predicate { UpdateFunc: func(event event.UpdateEvent) bool { if labelExists(InsightsLabel, event.ObjectNew) && !labelExists(InsightsWriteDeferred, event.ObjectNew) && - !labelExists(InsightsCMErrorLabel, event.ObjectNew) && // We don't want to respond to errored out CMs !insightsProcessedAnnotationExists(event.ObjectNew) { return true } From 38b078d8c2e1d8cb369eeb91b8a4657c72ee7085 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 11:00:03 +1300 Subject: [PATCH 3/6] Adding some arbitrary logging information --- controllers/configmap_controller.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index 4fedbde..b9a5c59 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "errors" "fmt" "k8s.io/apimachinery/pkg/types" "strconv" @@ -98,13 +99,16 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( insightsType := "unclassified" if _, ok := labels["insights.lagoon.sh/type"]; ok { insightsType = labels["insights.lagoon.sh/type"] + log.Info(fmt.Sprintf("Found insights.lagoon.sh/type:%v", insightsType)) } else { // insightsType can be determined by the incoming data if _, ok := labels["lagoon.sh/insightsType"]; ok { switch labels["lagoon.sh/insightsType"] { case ("sbom-gz"): + log.Info("Inferring insights type of sbom") insightsType = "sbom" case ("image-gz"): + log.Info("Inferring insights type of inspect") insightsType = "inspect" } } @@ -113,7 +117,8 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // We reject any type that isn't either sbom or inspect to restrict outgoing types if insightsType != "sbom" && insightsType != "inspect" { //// we mark this configMap as bad, and log an error - log.Error(nil, fmt.Sprintf("insightsType '%v' unrecognized - rejecting configMap", insightsType)) + err := errors.New(fmt.Sprintf("insightsType '%v' unrecognized - rejecting configMap", insightsType)) + log.Error(err, err.Error()) } else { // Here we attempt to process types using the new name structure From 16df6d4bdcf9b9404c758846edd94c307a3b5f8c Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 11:36:27 +1300 Subject: [PATCH 4/6] Fixes thinko and gets the labels from the right place --- controllers/configmap_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index b9a5c59..211b1d4 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -97,13 +97,13 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // insightsType is a way for us to classify incoming insights data, passing insightsType := "unclassified" - if _, ok := labels["insights.lagoon.sh/type"]; ok { - insightsType = labels["insights.lagoon.sh/type"] + if _, ok := configMap.Labels["insights.lagoon.sh/type"]; ok { + insightsType = configMap.Labels["insights.lagoon.sh/type"] log.Info(fmt.Sprintf("Found insights.lagoon.sh/type:%v", insightsType)) } else { // insightsType can be determined by the incoming data if _, ok := labels["lagoon.sh/insightsType"]; ok { - switch labels["lagoon.sh/insightsType"] { + switch configMap.Labels["lagoon.sh/insightsType"] { case ("sbom-gz"): log.Info("Inferring insights type of sbom") insightsType = "sbom" From 1e7dc78cd83a26e72d0e79f9302b697d703b6a56 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 11:56:10 +1300 Subject: [PATCH 5/6] some debugging info --- controllers/configmap_controller.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index 211b1d4..7df77d6 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -96,6 +96,13 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // insightsType is a way for us to classify incoming insights data, passing + + log.Info("incoming labels") + for k, v := range configMap.Labels { + log.Info(fmt.Sprintf("%v:%v\n", k, v)) + } + log.Info("incoming labels - end") + insightsType := "unclassified" if _, ok := configMap.Labels["insights.lagoon.sh/type"]; ok { insightsType = configMap.Labels["insights.lagoon.sh/type"] From f4452d194b184d66b1792d970fc89d4862f7bd5a Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 26 Feb 2024 13:08:07 +1300 Subject: [PATCH 6/6] Fixes label reference --- controllers/configmap_controller.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/controllers/configmap_controller.go b/controllers/configmap_controller.go index 7df77d6..d98dfd5 100644 --- a/controllers/configmap_controller.go +++ b/controllers/configmap_controller.go @@ -96,20 +96,13 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // insightsType is a way for us to classify incoming insights data, passing - - log.Info("incoming labels") - for k, v := range configMap.Labels { - log.Info(fmt.Sprintf("%v:%v\n", k, v)) - } - log.Info("incoming labels - end") - insightsType := "unclassified" if _, ok := configMap.Labels["insights.lagoon.sh/type"]; ok { insightsType = configMap.Labels["insights.lagoon.sh/type"] log.Info(fmt.Sprintf("Found insights.lagoon.sh/type:%v", insightsType)) } else { // insightsType can be determined by the incoming data - if _, ok := labels["lagoon.sh/insightsType"]; ok { + if _, ok := configMap.Labels["lagoon.sh/insightsType"]; ok { switch configMap.Labels["lagoon.sh/insightsType"] { case ("sbom-gz"): log.Info("Inferring insights type of sbom")