From c4a617fbe271754ba70bb68888b7b707b698e10a Mon Sep 17 00:00:00 2001 From: Amit Kumar Das Date: Fri, 8 Sep 2023 14:33:18 +0530 Subject: [PATCH] fix: [chaos-center]: Fixed subscriber status issue and minor experiment types fix in graphql server (#4158) * feat:[chaos-center]: Updated subscriber status changes Signed-off-by: Amit Kumar Das * feat:[chaos-center]: Updated types and chaosdata result fix in gql server Signed-off-by: Amit Kumar Das --------- Signed-off-by: Amit Kumar Das --- chaoscenter/graphql/server/graph/resolver.go | 4 +- .../pkg/chaos_experiment/handler/handler.go | 2 +- .../handler/handler.go | 13 +++-- .../service.go | 27 +++++---- .../types.go | 24 ++++---- .../mongodb/chaos_experiment/schema.go | 1 + chaoscenter/subscriber/pkg/events/util.go | 3 +- chaoscenter/subscriber/pkg/events/workflow.go | 58 ++++++++++++++++--- chaoscenter/subscriber/pkg/types/event.go | 9 +++ 9 files changed, 97 insertions(+), 44 deletions(-) rename chaoscenter/graphql/server/pkg/{choas_experiment_run => chaos_experiment_run}/handler/handler.go (99%) rename chaoscenter/graphql/server/pkg/{choas_experiment_run => chaos_experiment_run}/service.go (86%) rename chaoscenter/graphql/server/pkg/{choas_experiment_run => chaos_experiment_run}/types.go (86%) diff --git a/chaoscenter/graphql/server/graph/resolver.go b/chaoscenter/graphql/server/graph/resolver.go index d2d8f768cbd..e44c7ee378f 100644 --- a/chaoscenter/graphql/server/graph/resolver.go +++ b/chaoscenter/graphql/server/graph/resolver.go @@ -9,10 +9,10 @@ import ( "github.com/litmuschaos/litmus/chaoscenter/graphql/server/graph/generated" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/authorization" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment/handler" + chaos_experiment_run2 "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run" + runHandler "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run/handler" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_infrastructure" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaoshub" - chaos_experiment_run2 "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run" - runHandler "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run/handler" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment" "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment_run" diff --git a/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go b/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go index abc010032db..6c11db6e93b 100644 --- a/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go +++ b/chaoscenter/graphql/server/pkg/chaos_experiment/handler/handler.go @@ -28,7 +28,7 @@ import ( "go.mongodb.org/mongo-driver/mongo" types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment" - chaosExperimentRun "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run" + chaosExperimentRun "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run" store "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/data-store" dbChaosExperiment "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment" diff --git a/chaoscenter/graphql/server/pkg/choas_experiment_run/handler/handler.go b/chaoscenter/graphql/server/pkg/chaos_experiment_run/handler/handler.go similarity index 99% rename from chaoscenter/graphql/server/pkg/choas_experiment_run/handler/handler.go rename to chaoscenter/graphql/server/pkg/chaos_experiment_run/handler/handler.go index 536b77f7c8a..08f6fb01ac8 100644 --- a/chaoscenter/graphql/server/pkg/choas_experiment_run/handler/handler.go +++ b/chaoscenter/graphql/server/pkg/chaos_experiment_run/handler/handler.go @@ -35,7 +35,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" - types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/choas_experiment_run" + types "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/chaos_experiment_run" store "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/data-store" dbChaosExperiment "github.com/litmuschaos/litmus/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment" @@ -1222,6 +1222,7 @@ func (c *ChaosExperimentRunHandler) ChaosExperimentRunEvent(event model.Experime {"recent_experiment_run_details.$.phase", executionData.Phase}, {"recent_experiment_run_details.$.completed", event.Completed}, {"recent_experiment_run_details.$.experiment_run_id", event.ExperimentRunID}, + {"recent_experiment_run_details.$.probes", probes}, {"recent_experiment_run_details.$.resiliency_score", workflowRunMetrics.ResiliencyScore}, {"recent_experiment_run_details.$.updated_at", currentTime.UnixMilli()}, {"recent_experiment_run_details.$.updated_by", string(updatedBy)}, @@ -1244,11 +1245,11 @@ func (c *ChaosExperimentRunHandler) ChaosExperimentRunEvent(event model.Experime NotifyID: event.NotifyID, Phase: executionData.Phase, ResiliencyScore: &workflowRunMetrics.ResiliencyScore, - FaultsPassed: &workflowRunMetrics.FaultsPassed, - FaultsFailed: &workflowRunMetrics.FaultsFailed, - FaultsAwaited: &workflowRunMetrics.FaultsAwaited, - FaultsStopped: &workflowRunMetrics.FaultsStopped, - FaultsNA: &workflowRunMetrics.FaultsNA, + FaultsPassed: &workflowRunMetrics.ExperimentsPassed, + FaultsFailed: &workflowRunMetrics.ExperimentsFailed, + FaultsAwaited: &workflowRunMetrics.ExperimentsAwaited, + FaultsStopped: &workflowRunMetrics.ExperimentsStopped, + FaultsNA: &workflowRunMetrics.ExperimentsNA, TotalFaults: &workflowRunMetrics.TotalExperiments, ExecutionData: string(exeData), RevisionID: event.RevisionID, diff --git a/chaoscenter/graphql/server/pkg/choas_experiment_run/service.go b/chaoscenter/graphql/server/pkg/chaos_experiment_run/service.go similarity index 86% rename from chaoscenter/graphql/server/pkg/choas_experiment_run/service.go rename to chaoscenter/graphql/server/pkg/chaos_experiment_run/service.go index 8434c8e6ee6..5cb6976b003 100644 --- a/chaoscenter/graphql/server/pkg/choas_experiment_run/service.go +++ b/chaoscenter/graphql/server/pkg/chaos_experiment_run/service.go @@ -68,17 +68,17 @@ func (c *chaosExperimentRunService) ProcessExperimentRunDelete(ctx context.Conte // ProcessCompletedExperimentRun calculates the Resiliency Score and returns the updated ExecutionData func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData ExecutionData, wfID string, runID string) (ExperimentRunMetrics, error) { - var weightSum, totalTestResult = 0, 0 + weightSum, totalTestResult := 0, 0 var result ExperimentRunMetrics weightMap := map[string]int{} - chaosExperiments, err := c.chaosExperimentOperator.GetExperiment(context.TODO(), bson.D{ + chaosWorkflows, err := c.chaosExperimentOperator.GetExperiment(context.TODO(), bson.D{ {"experiment_id", wfID}, }) if err != nil { return result, fmt.Errorf("failed to get experiment from db on complete, error: %w", err) } - for _, rev := range chaosExperiments.Revision { + for _, rev := range chaosWorkflows.Revision { if rev.RevisionID == execData.RevisionID { for _, weights := range rev.Weightages { weightMap[weights.FaultName] = weights.Weightage @@ -89,7 +89,6 @@ func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData Execu } result.TotalExperiments = len(weightMap) - for _, value := range execData.Nodes { if value.Type == "ChaosEngine" { experimentName := "" @@ -108,20 +107,20 @@ func (c *chaosExperimentRunService) ProcessCompletedExperimentRun(execData Execu x, _ := strconv.Atoi(value.ChaosExp.ProbeSuccessPercentage) totalTestResult += weight * x } - if value.ChaosExp.FaultVerdict == "Pass" { - result.FaultsPassed += 1 + if value.ChaosExp.ExperimentVerdict == "Pass" { + result.ExperimentsPassed += 1 } - if value.ChaosExp.FaultVerdict == "Fail" { - result.FaultsFailed += 1 + if value.ChaosExp.ExperimentVerdict == "Fail" { + result.ExperimentsFailed += 1 } - if value.ChaosExp.FaultVerdict == "Awaited" { - result.FaultsAwaited += 1 + if value.ChaosExp.ExperimentVerdict == "Awaited" { + result.ExperimentsAwaited += 1 } - if value.ChaosExp.FaultVerdict == "Stopped" { - result.FaultsStopped += 1 + if value.ChaosExp.ExperimentVerdict == "Stopped" { + result.ExperimentsStopped += 1 } - if value.ChaosExp.FaultVerdict == "N/A" || value.ChaosExp.FaultVerdict == "" { - result.FaultsNA += 1 + if value.ChaosExp.ExperimentVerdict == "N/A" || value.ChaosExp.ExperimentVerdict == "" { + result.ExperimentsNA += 1 } } } diff --git a/chaoscenter/graphql/server/pkg/choas_experiment_run/types.go b/chaoscenter/graphql/server/pkg/chaos_experiment_run/types.go similarity index 86% rename from chaoscenter/graphql/server/pkg/choas_experiment_run/types.go rename to chaoscenter/graphql/server/pkg/chaos_experiment_run/types.go index 6229d1c60ba..cc46812762b 100644 --- a/chaoscenter/graphql/server/pkg/choas_experiment_run/types.go +++ b/chaoscenter/graphql/server/pkg/chaos_experiment_run/types.go @@ -5,13 +5,13 @@ import ( ) type ExperimentRunMetrics struct { - ResiliencyScore float64 `json:"resiliency_score"` - FaultsPassed int `json:"faults_passed"` - FaultsFailed int `json:"faults_failed"` - FaultsAwaited int `json:"faults_awaited"` - FaultsStopped int `json:"faults_stopped"` - FaultsNA int `json:"experiments_na"` - TotalExperiments int `json:"total_faults"` + ResiliencyScore float64 `json:"resiliency_score"` + ExperimentsPassed int `json:"experiments_passed"` + ExperimentsFailed int `json:"experiments_failed"` + ExperimentsAwaited int `json:"experiments_awaited"` + ExperimentsStopped int `json:"experiments_stopped"` + ExperimentsNA int `json:"experiments_na"` + TotalExperiments int `json:"total_experiments"` } type ExecutionData struct { @@ -39,19 +39,19 @@ type Node struct { FinishedAt string `json:"finishedAt"` Children []string `json:"children"` Type string `json:"type"` - ChaosExp *ChaosData `json:"chaos_data,omitempty"` + ChaosExp *ChaosData `json:"chaosData,omitempty"` } // ChaosData is the data we get from chaos exporter type ChaosData struct { EngineUID string `json:"engineUID"` - EngineContext string `json:"engineContext"` + EngineContext string `json:"engine_context"` EngineName string `json:"engineName"` Namespace string `json:"namespace"` - FaultName string `json:"faultName"` - FaultStatus string `json:"faultStatus"` + ExperimentName string `json:"experimentName"` + ExperimentStatus string `json:"experimentStatus"` LastUpdatedAt string `json:"lastUpdatedAt"` - FaultVerdict string `json:"faultVerdict"` + ExperimentVerdict string `json:"experimentVerdict"` ExperimentPod string `json:"experimentPod"` RunnerPod string `json:"runnerPod"` ProbeSuccessPercentage string `json:"probeSuccessPercentage"` diff --git a/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go b/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go index cfb2a60f706..f79637995da 100644 --- a/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go +++ b/chaoscenter/graphql/server/pkg/database/mongodb/chaos_experiment/schema.go @@ -24,6 +24,7 @@ type ExperimentRunDetail struct { ResiliencyScore *float64 `bson:"resiliency_score,omitempty"` Completed bool `bson:"completed"` RunSequence int `bson:"run_sequence"` + Probe []Probes `bson:"probes"` } // ChaosExperimentRequest contains the required fields to be stored in the database for a chaos experiment input diff --git a/chaoscenter/subscriber/pkg/events/util.go b/chaoscenter/subscriber/pkg/events/util.go index dc2506e5779..35ff5e4e0b1 100644 --- a/chaoscenter/subscriber/pkg/events/util.go +++ b/chaoscenter/subscriber/pkg/events/util.go @@ -52,7 +52,7 @@ func getChaosData(nodeStatus v1alpha13.NodeStatus, engineName, engineNS string, // considering chaosengine will only have 1 experiment cd.ExperimentPod = crd.Status.Experiments[0].ExpPod cd.RunnerPod = crd.Status.Experiments[0].Runner - cd.ExperimentStatus = string(crd.Status.Experiments[0].Status) + cd.ExperimentStatus = string(crd.Status.EngineStatus) cd.ExperimentName = crd.Status.Experiments[0].Name cd.LastUpdatedAt = strconv.FormatInt(crd.Status.Experiments[0].LastUpdateTime.Unix(), 10) cd.ExperimentVerdict = crd.Status.Experiments[0].Verdict @@ -73,6 +73,7 @@ func getChaosData(nodeStatus v1alpha13.NodeStatus, engineName, engineNS string, if expRes.Status.ExperimentStatus.ErrorOutput != nil { cd.FailStep = fmt.Sprintf("%s : %s", expRes.Status.ExperimentStatus.ErrorOutput.ErrorCode, expRes.Status.ExperimentStatus.ErrorOutput.Reason) } + cd.ExperimentStatus = string(expRes.Status.ExperimentStatus.Phase) } return cd, nil } diff --git a/chaoscenter/subscriber/pkg/events/workflow.go b/chaoscenter/subscriber/pkg/events/workflow.go index 4036a925277..ce102d1d249 100644 --- a/chaoscenter/subscriber/pkg/events/workflow.go +++ b/chaoscenter/subscriber/pkg/events/workflow.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "strconv" - "strings" "time" "subscriber/pkg/graphql" @@ -154,13 +153,8 @@ func WorkflowEventHandler(workflowObj *v1alpha1.Workflow, eventType string, star ChaosExp: cd, Message: nodeStatus.Message, } - if cd != nil && strings.ToLower(cd.ExperimentVerdict) == "fail" { - details.Phase = "Failed" - details.Message = "Chaos Experiment Failed" - cd.ExperimentVerdict = "Fail" - } else if cd != nil && strings.ToLower(cd.ExperimentVerdict) == "pass" { - details.Phase = "Passed" - cd.ExperimentVerdict = "Pass" + if nodeType == "ChaosEngine" && cd != nil { + details.Phase = cd.ExperimentStatus } nodes[nodeStatus.ID] = details } @@ -227,6 +221,11 @@ func SendWorkflowUpdates(infraData map[string]string, event types.WorkflowEvent) } } } + // Setting up the experiment status + // based on different probes results + // present in the experiment + event.Phase = getExperimentStatus(event) + eventMap[event.UID] = event // generate graphql payload @@ -276,3 +275,46 @@ func updateWorkflowStatus(status v1alpha1.WorkflowPhase) string { return "Pending" } } + +// getExperimentStatus is used to fetch the final experiment status +// based on the fault/probe status +func getExperimentStatus(experiment types.WorkflowEvent) string { + var ( + errorCount = 0 + completedWithProbeFailureCount = 0 + status = experiment.Phase + ) + + // Once the workflow is completed, and it is not stopped, + // we will fetch the data based on the different + // node statuses(which are coming from the probe status + // of these faults) + if status == "Stopped" || experiment.FinishedAt == "" { + return status + } + + for _, node := range experiment.Nodes { + if node.Type == "ChaosEngine" && node.ChaosExp == nil { + errorCount++ + continue + } + switch node.Phase { + case string(types.FaultCompletedWithProbeFailure): + completedWithProbeFailureCount++ + case string(types.Error): + errorCount++ + } + + } + + // For multiple faults, if one of the fault + // errors out, priority is given to the error + // status and then the remaining status + if errorCount > 0 { + status = string(types.Error) + } else if completedWithProbeFailureCount > 0 { + status = string(types.FaultCompletedWithProbeFailure) + } + + return status +} diff --git a/chaoscenter/subscriber/pkg/types/event.go b/chaoscenter/subscriber/pkg/types/event.go index a376ec52bf5..b5b230fd8e0 100644 --- a/chaoscenter/subscriber/pkg/types/event.go +++ b/chaoscenter/subscriber/pkg/types/event.go @@ -49,3 +49,12 @@ type ChaosData struct { FailStep string `json:"failStep"` ChaosResult *v1alpha1.ChaosResult `json:"chaosResult"` } + +type FaultStatus string + +const ( + FaultCompleted FaultStatus = "Completed" + FaultCompletedWithProbeFailure FaultStatus = "Completed_With_Probe_Failure" + Error FaultStatus = "Error" + Stopped FaultStatus = "Stopped" +)