fix: [chaos-center]: Fixed subscriber status issue and minor experime…

…nt types fix in graphql server (#4158) * feat:[chaos-center]: Updated subscriber status changes Signed-off-by: Amit Kumar Das <[email protected]> * feat:[chaos-center]: Updated types and chaosdata result fix in gql server Signed-off-by: Amit Kumar Das <[email protected]> --------- Signed-off-by: Amit Kumar Das <[email protected]>
litmuschaos · Sep 8, 2023 · c4a617f · c4a617f
1 parent c184c3e
commit c4a617f
Show file tree

Hide file tree

Showing 9 changed files with 97 additions and 44 deletions.
diff --git a/chaoscenter/graphql/server/graph/resolver.go b/chaoscenter/graphql/server/graph/resolver.go
@@ -9,10 +9,10 @@ import (
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/graph/generated"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/authorization"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment/handler"
+	chaos_experiment_run2 "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run"
+	runHandler "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run/handler"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_infrastructure"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaoshub"
-	chaos_experiment_run2 "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run"
-	runHandler "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run/handler"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment"
 	"github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment_run"

diff --git a/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go b/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go
@@ -28,7 +28,7 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 
 	types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment"
-	chaosExperimentRun "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run"
+	chaosExperimentRun "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run"
 	store "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/data-store"
 	dbChaosExperiment "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment"
 

diff --git a/...g/choas_experiment_run/handler/handler.go → ...g/chaos_experiment_run/handler/handler.go b/...g/choas_experiment_run/handler/handler.go → ...g/chaos_experiment_run/handler/handler.go
@@ -35,7 +35,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 
-	types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run"
+	types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run"
 	store "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/data-store"
 	dbChaosExperiment "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment"
 
@@ -1222,6 +1222,7 @@ func (c *ChaosExperimentRunHandler) ChaosExperimentRunEvent(event model.Experime
 						{"recent_experiment_run_details.$.phase", executionData.Phase},
 						{"recent_experiment_run_details.$.completed", event.Completed},
 						{"recent_experiment_run_details.$.experiment_run_id", event.ExperimentRunID},
+						{"recent_experiment_run_details.$.probes", probes},
 						{"recent_experiment_run_details.$.resiliency_score", workflowRunMetrics.ResiliencyScore},
 						{"recent_experiment_run_details.$.updated_at", currentTime.UnixMilli()},
 						{"recent_experiment_run_details.$.updated_by", string(updatedBy)},
@@ -1244,11 +1245,11 @@ func (c *ChaosExperimentRunHandler) ChaosExperimentRunEvent(event model.Experime
 			NotifyID:        event.NotifyID,
 			Phase:           executionData.Phase,
 			ResiliencyScore: &workflowRunMetrics.ResiliencyScore,
-			FaultsPassed:    &workflowRunMetrics.FaultsPassed,
-			FaultsFailed:    &workflowRunMetrics.FaultsFailed,
-			FaultsAwaited:   &workflowRunMetrics.FaultsAwaited,
-			FaultsStopped:   &workflowRunMetrics.FaultsStopped,
-			FaultsNA:        &workflowRunMetrics.FaultsNA,
+			FaultsPassed:    &workflowRunMetrics.ExperimentsPassed,
+			FaultsFailed:    &workflowRunMetrics.ExperimentsFailed,
+			FaultsAwaited:   &workflowRunMetrics.ExperimentsAwaited,
+			FaultsStopped:   &workflowRunMetrics.ExperimentsStopped,
+			FaultsNA:        &workflowRunMetrics.ExperimentsNA,
 			TotalFaults:     &workflowRunMetrics.TotalExperiments,
 			ExecutionData:   string(exeData),
 			RevisionID:      event.RevisionID,

diff --git a/...erver/pkg/choas_experiment_run/service.go → ...erver/pkg/chaos_experiment_run/service.go b/...erver/pkg/choas_experiment_run/service.go → ...erver/pkg/chaos_experiment_run/service.go
@@ -68,17 +68,17 @@ func (c *chaosExperimentRunService) ProcessExperimentRunDelete(ctx context.Conte
 
 // ProcessCompletedExperimentRun calculates the Resiliency Score and returns the updated ExecutionData
 func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData ExecutionData, wfID string, runID string) (ExperimentRunMetrics, error) {
-	var weightSum, totalTestResult = 0, 0
+	weightSum, totalTestResult := 0, 0
 	var result ExperimentRunMetrics
 	weightMap := map[string]int{}
 
-	chaosExperiments, err := c.chaosExperimentOperator.GetExperiment(context.TODO(), bson.D{
+	chaosWorkflows, err := c.chaosExperimentOperator.GetExperiment(context.TODO(), bson.D{
 		{"experiment_id", wfID},
 	})
 	if err != nil {
 		return result, fmt.Errorf("failed to get experiment from db on complete, error: %w", err)
 	}
-	for _, rev := range chaosExperiments.Revision {
+	for _, rev := range chaosWorkflows.Revision {
 		if rev.RevisionID == execData.RevisionID {
 			for _, weights := range rev.Weightages {
 				weightMap[weights.FaultName] = weights.Weightage
@@ -89,7 +89,6 @@ func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData Execu
 	}
 
 	result.TotalExperiments = len(weightMap)
-
 	for _, value := range execData.Nodes {
 		if value.Type == "ChaosEngine" {
 			experimentName := ""
@@ -108,20 +107,20 @@ func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData Execu
 				x, _ := strconv.Atoi(value.ChaosExp.ProbeSuccessPercentage)
 				totalTestResult += weight * x
 			}
-			if value.ChaosExp.FaultVerdict == "Pass" {
-				result.FaultsPassed += 1
+			if value.ChaosExp.ExperimentVerdict == "Pass" {
+				result.ExperimentsPassed += 1
 			}
-			if value.ChaosExp.FaultVerdict == "Fail" {
-				result.FaultsFailed += 1
+			if value.ChaosExp.ExperimentVerdict == "Fail" {
+				result.ExperimentsFailed += 1
 			}
-			if value.ChaosExp.FaultVerdict == "Awaited" {
-				result.FaultsAwaited += 1
+			if value.ChaosExp.ExperimentVerdict == "Awaited" {
+				result.ExperimentsAwaited += 1
 			}
-			if value.ChaosExp.FaultVerdict == "Stopped" {
-				result.FaultsStopped += 1
+			if value.ChaosExp.ExperimentVerdict == "Stopped" {
+				result.ExperimentsStopped += 1
 			}
-			if value.ChaosExp.FaultVerdict == "N/A" || value.ChaosExp.FaultVerdict == "" {
-				result.FaultsNA += 1
+			if value.ChaosExp.ExperimentVerdict == "N/A" || value.ChaosExp.ExperimentVerdict == "" {
+				result.ExperimentsNA += 1
 			}
 		}
 	}

diff --git a/.../server/pkg/choas_experiment_run/types.go → .../server/pkg/chaos_experiment_run/types.go b/.../server/pkg/choas_experiment_run/types.go → .../server/pkg/chaos_experiment_run/types.go
@@ -5,13 +5,13 @@ import (
 )
 
 type ExperimentRunMetrics struct {
-	ResiliencyScore  float64 `json:"resiliency_score"`
-	FaultsPassed     int     `json:"faults_passed"`
-	FaultsFailed     int     `json:"faults_failed"`
-	FaultsAwaited    int     `json:"faults_awaited"`
-	FaultsStopped    int     `json:"faults_stopped"`
-	FaultsNA         int     `json:"experiments_na"`
-	TotalExperiments int     `json:"total_faults"`
+	ResiliencyScore    float64 `json:"resiliency_score"`
+	ExperimentsPassed  int     `json:"experiments_passed"`
+	ExperimentsFailed  int     `json:"experiments_failed"`
+	ExperimentsAwaited int     `json:"experiments_awaited"`
+	ExperimentsStopped int     `json:"experiments_stopped"`
+	ExperimentsNA      int     `json:"experiments_na"`
+	TotalExperiments   int     `json:"total_experiments"`
 }
 
 type ExecutionData struct {
@@ -39,19 +39,19 @@ type Node struct {
 	FinishedAt string     `json:"finishedAt"`
 	Children   []string   `json:"children"`
 	Type       string     `json:"type"`
-	ChaosExp   *ChaosData `json:"chaos_data,omitempty"`
+	ChaosExp   *ChaosData `json:"chaosData,omitempty"`
 }
 
 // ChaosData is the data we get from chaos exporter
 type ChaosData struct {
 	EngineUID              string                  `json:"engineUID"`
-	EngineContext          string                  `json:"engineContext"`
+	EngineContext          string                  `json:"engine_context"`
 	EngineName             string                  `json:"engineName"`
 	Namespace              string                  `json:"namespace"`
-	FaultName              string                  `json:"faultName"`
-	FaultStatus            string                  `json:"faultStatus"`
+	ExperimentName         string                  `json:"experimentName"`
+	ExperimentStatus       string                  `json:"experimentStatus"`
 	LastUpdatedAt          string                  `json:"lastUpdatedAt"`
-	FaultVerdict           string                  `json:"faultVerdict"`
+	ExperimentVerdict      string                  `json:"experimentVerdict"`
 	ExperimentPod          string                  `json:"experimentPod"`
 	RunnerPod              string                  `json:"runnerPod"`
 	ProbeSuccessPercentage string                  `json:"probeSuccessPercentage"`

diff --git a/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go b/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go
@@ -24,6 +24,7 @@ type ExperimentRunDetail struct {
 	ResiliencyScore *float64 `bson:"resiliency_score,omitempty"`
 	Completed       bool     `bson:"completed"`
 	RunSequence     int      `bson:"run_sequence"`
+	Probe           []Probes `bson:"probes"`
 }
 
 // ChaosExperimentRequest contains the required fields to be stored in the database for a chaos experiment input

diff --git a/chaoscenter/subscriber/pkg/events/util.go b/chaoscenter/subscriber/pkg/events/util.go
@@ -52,7 +52,7 @@ func getChaosData(nodeStatus v1alpha13.NodeStatus, engineName, engineNS string,
 	// considering chaosengine will only have 1 experiment
 	cd.ExperimentPod = crd.Status.Experiments[0].ExpPod
 	cd.RunnerPod = crd.Status.Experiments[0].Runner
-	cd.ExperimentStatus = string(crd.Status.Experiments[0].Status)
+	cd.ExperimentStatus = string(crd.Status.EngineStatus)
 	cd.ExperimentName = crd.Status.Experiments[0].Name
 	cd.LastUpdatedAt = strconv.FormatInt(crd.Status.Experiments[0].LastUpdateTime.Unix(), 10)
 	cd.ExperimentVerdict = crd.Status.Experiments[0].Verdict
@@ -73,6 +73,7 @@ func getChaosData(nodeStatus v1alpha13.NodeStatus, engineName, engineNS string,
 		if expRes.Status.ExperimentStatus.ErrorOutput != nil {
 			cd.FailStep = fmt.Sprintf("%s : %s", expRes.Status.ExperimentStatus.ErrorOutput.ErrorCode, expRes.Status.ExperimentStatus.ErrorOutput.Reason)
 		}
+		cd.ExperimentStatus = string(expRes.Status.ExperimentStatus.Phase)
 	}
 	return cd, nil
 }

diff --git a/chaoscenter/subscriber/pkg/events/workflow.go b/chaoscenter/subscriber/pkg/events/workflow.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-	"strings"
 	"time"
 
 	"subscriber/pkg/graphql"
@@ -154,13 +153,8 @@ func WorkflowEventHandler(workflowObj *v1alpha1.Workflow, eventType string, star
 			ChaosExp:   cd,
 			Message:    nodeStatus.Message,
 		}
-		if cd != nil && strings.ToLower(cd.ExperimentVerdict) == "fail" {
-			details.Phase = "Failed"
-			details.Message = "Chaos Experiment Failed"
-			cd.ExperimentVerdict = "Fail"
-		} else if cd != nil && strings.ToLower(cd.ExperimentVerdict) == "pass" {
-			details.Phase = "Passed"
-			cd.ExperimentVerdict = "Pass"
+		if nodeType == "ChaosEngine" && cd != nil {
+			details.Phase = cd.ExperimentStatus
 		}
 		nodes[nodeStatus.ID] = details
 	}
@@ -227,6 +221,11 @@ func SendWorkflowUpdates(infraData map[string]string, event types.WorkflowEvent)
 			}
 		}
 	}
+	// Setting up the experiment status
+	// based on different probes results
+	// present in the experiment
+	event.Phase = getExperimentStatus(event)
+
 	eventMap[event.UID] = event
 
 	// generate graphql payload
@@ -276,3 +275,46 @@ func updateWorkflowStatus(status v1alpha1.WorkflowPhase) string {
 		return "Pending"
 	}
 }
+
+// getExperimentStatus is used to fetch the final experiment status
+// based on the fault/probe status
+func getExperimentStatus(experiment types.WorkflowEvent) string {
+	var (
+		errorCount                     = 0
+		completedWithProbeFailureCount = 0
+		status                         = experiment.Phase
+	)
+
+	// Once the workflow is completed, and it is not stopped,
+	// we will fetch the data based on the different
+	// node statuses(which are coming from the probe status
+	// of these faults)
+	if status == "Stopped" || experiment.FinishedAt == "" {
+		return status
+	}
+
+	for _, node := range experiment.Nodes {
+		if node.Type == "ChaosEngine" && node.ChaosExp == nil {
+			errorCount++
+			continue
+		}
+		switch node.Phase {
+		case string(types.FaultCompletedWithProbeFailure):
+			completedWithProbeFailureCount++
+		case string(types.Error):
+			errorCount++
+		}
+
+	}
+
+	// For multiple faults, if one of the fault
+	// errors out, priority is given to the error
+	// status and then the remaining status
+	if errorCount > 0 {
+		status = string(types.Error)
+	} else if completedWithProbeFailureCount > 0 {
+		status = string(types.FaultCompletedWithProbeFailure)
+	}
+
+	return status
+}
diff --git a/chaoscenter/subscriber/pkg/types/event.go b/chaoscenter/subscriber/pkg/types/event.go
@@ -49,3 +49,12 @@ type ChaosData struct {
 	FailStep               string                `json:"failStep"`
 	ChaosResult            *v1alpha1.ChaosResult `json:"chaosResult"`
 }
+
+type FaultStatus string
+
+const (
+	FaultCompleted                 FaultStatus = "Completed"
+	FaultCompletedWithProbeFailure FaultStatus = "Completed_With_Probe_Failure"
+	Error                          FaultStatus = "Error"
+	Stopped                        FaultStatus = "Stopped"
+)