fix: autoscale error handling (#5481)

* fix: autoscale error handling * minor updates * chore: renaming and logging improved * chore: logging * chore: k8s request adapters * chore: k8s adapter * chore: updated common-lib version * fix: 5xx fixes issues and Panic handling (#5402) * feat: 5xx issue for manual cd trigger API * fix: panic handling for case scantool not found * fix: switch ci fixes (#5358) * fix: switch ci fixes * chore: review refactoring * chore: refactoring * fix: add missing validations in switching to external ci * fix: ci pipeline id is not being updated in cd pipeline if switching from external ci to other ci types * fix: update ci pipeline id in all the cd workflows that exists in the given workflow * fix: checkIfNsExistsForEnvIds , empty envIds check fix * fix: ci patch rbac fixes (#5461) * fix: ci patch rbac fixes * fix: rbac optimisation * fix: refactoring --------- Co-authored-by: Asutosh Das <[email protected]> * feat: deployment config migration (#5368) * inital commit - deployment config service * wip: migration changes for devtron apps * changes for helm apps * dev testing fixes * wire fix * bulk get function * wip: bulk logic fixes * wip: bulk fixes * feat: Cd pipeline migration sql and query usage update (#5416) * Added sql * removed unused mthod * wip * wip * wip * wip * wip * wip * QUERY : updated GetOfLatestCdWfrByCdPipelineId * QUERY : GetOfLatestInstalledAppVersionHistoryByInstalledAppVersionId * QUERY : fix GetOfLatestCdWfrByCdPipelineId * QUERY : fix GetOfLatestInstalledAppVersionHistoryByInstalledAppVersionId * QUERY : updated GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses * QUERY : updated GetArgoPipelineByArgoAppName * QUERY : updated GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses * QUERY : updated GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines * QUERY : updated GetAppAndEnvDetailsForDeploymentAppTypePipeline * QUERY : updated FindActiveByEnvIdAndDeploymentType * QUERY : updated GetActiveInstalledAppByEnvIdAndDeploymentType * QUERY : updated GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore * QUERY : updated GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatusesForAppStore * QUERY : updated GetInstalledAppByInstalledAppVersionId * QUERY : updated GetInstalledAppByAppIdAndDeploymentType * QUERY : updated GetAppAndEnvDetailsForDeploymentAppTypeInstalledApps * QUERY : updated FindLatestByAppIdAndEnvId * QUERY : updated GetInstalledAppByGitRepoUrl * QUERY : updated FindChartByGitRepoUrl * dev testing fixes * QUERY : UPDATED buildAppListingWhereCondition and FIXED deploymentDetailsByAppIdAndEnvId * function for bulk update config * query changes * remaining query changes * PR review changes and logical fixes * nil pointer fix * fix cd pipeline helm app create * dev testing fixes * updating app stages status check * query active check for deployment config * wip * adding flag for migration and removing bulk code * build fix * wip: modifying get function * handling custom gitops case for app clone * handling app clone with custom gitops for system generated url * fix: wrong call for helm app deployment app type change * fix migration function for helm app * wip: updating dc on template update * fix: replacing migrate with get call * fix: wrong call for helm app * minor fixes * renaming chart path * updating deprecated columns * updating dc.active * updating dc.active * fix redirection * removing chart path * updating migration * conditions in patch object * fix error in flag off-> custom-gitops-> app-clone-> deploy-> flag-on-> migrate * wire fix * audit log fix * migation update * reverted 107 sql newline add --------- Co-authored-by: kartik-579 <[email protected]> Co-authored-by: kartik-579 <[email protected]> * handle nil check (#5497) * doc: Added FAQ no. 28 + GoLang-migrate Link + Code Block Fix (#5502) * Added FAQ no. 28 + GoLang-migrate Link + Code Block Fix * Removed extra spacing at EOF * fix: bitbucket commit race condition for concurrent requests (#5505) * fix: hotfix v0.14.0 fixes (#5500) * fix: patch ci rbac fix (#5498) * handle nil check (#5497) * fix: patch ci rbac fix --------- Co-authored-by: ayu-devtron <[email protected]> * fix: external ci creation fix (#5499) * handle nil check (#5497) * fix: external ci creation fix --------- Co-authored-by: ayu-devtron <[email protected]> * fix for deployment config - auto post cd not working (#5501) * fix: do not create deployment config for switch ci, if pipeline is already created. (#5506) * chore: main sync (#5510) * handle nil check (#5497) * doc: Added FAQ no. 28 + GoLang-migrate Link + Code Block Fix (#5502) * Added FAQ no. 28 + GoLang-migrate Link + Code Block Fix * Removed extra spacing at EOF * fix: bitbucket commit race condition for concurrent requests (#5505) --------- Co-authored-by: ayu-devtron <[email protected]> Co-authored-by: ashokdevtron <[email protected]> Co-authored-by: Asutosh Das <[email protected]> --------- Co-authored-by: ayu-devtron <[email protected]> Co-authored-by: kartik-579 <[email protected]> Co-authored-by: ashokdevtron <[email protected]> Co-authored-by: Asutosh Das <[email protected]> --------- Co-authored-by: Gireesh Naidu <[email protected]> Co-authored-by: iamayushm <[email protected]> Co-authored-by: kartik-579 <[email protected]> Co-authored-by: kartik-579 <[email protected]> Co-authored-by: ayu-devtron <[email protected]> Co-authored-by: ashokdevtron <[email protected]> * updated: common-lib version --------- Co-authored-by: Gireesh Naidu <[email protected]> Co-authored-by: iamayushm <[email protected]> Co-authored-by: kartik-579 <[email protected]> Co-authored-by: kartik-579 <[email protected]> Co-authored-by: ayu-devtron <[email protected]> Co-authored-by: ashokdevtron <[email protected]>
devtron-labs · Jul 18, 2024 · 21b7d83 · 21b7d83
1 parent 9db1489
commit 21b7d83
Show file tree

Hide file tree

Showing 13 changed files with 258 additions and 89 deletions.
diff --git a/client/argocdServer/k8sClient.go b/client/argocdServer/k8sClient.go
@@ -20,11 +20,14 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/devtron-labs/common-lib/utils/k8s"
+	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/cluster/repository"
 	"go.uber.org/zap"
 	"io/ioutil"
+	k8sError "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -117,7 +120,7 @@ func (impl ArgoK8sClientImpl) CreateArgoApplication(ctx context.Context, namespa
 	if err != nil {
 		return fmt.Errorf("error creating argo cd app")
 	}
-	impl.logger.Infow("creating application", "req", application)
+	impl.logger.Debugw("creating argo application resource", "application", application)
 	res, err := client.
 		Post().
 		Resource("applications").
@@ -126,23 +129,48 @@ func (impl ArgoK8sClientImpl) CreateArgoApplication(ctx context.Context, namespa
 		Do(ctx).Raw()
 
 	if err != nil {
-		response := make(map[string]interface{})
-		err := json.Unmarshal(res, &response)
-		if err != nil {
-			impl.logger.Errorw("unmarshal error on app update status", "err", err)
-			return fmt.Errorf("error creating argo cd app")
-		}
-		message := "error creating argo cd app"
-		if response != nil && response["message"] != nil {
-			message = response["message"].(string)
-		}
-		return fmt.Errorf(message)
+		impl.logger.Errorw("error in argo application resource creation", "namespace", namespace, "res", res, "err", err)
+		return impl.handleArgoAppCreationError(res, err)
 	}
 
-	impl.logger.Infow("argo app create res", "res", string(res), "err", err)
+	impl.logger.Infow("argo app create successfully", "namespace", namespace, "res", string(res))
 	return err
 }
 
+func (impl ArgoK8sClientImpl) handleArgoAppCreationError(res []byte, err error) error {
+	// default error set
+	apiError := &util.ApiError{
+		InternalMessage: "error creating argo cd app",
+		UserMessage:     "error creating argo cd app",
+	}
+	// error override for errors.StatusError
+	if statusError := (&k8sError.StatusError{}); errors.As(err, &statusError) {
+		apiError.HttpStatusCode = int(statusError.Status().Code)
+		apiError.InternalMessage = statusError.Error()
+		apiError.UserMessage = statusError.Error()
+	}
+	response := make(map[string]interface{})
+	jsonErr := json.Unmarshal(res, &response)
+	if jsonErr != nil {
+		impl.logger.Errorw("unmarshal error on app update status", "err", jsonErr)
+		return apiError
+	}
+	// error override if API response exists, as response errors are more readable
+	if response != nil {
+		if statusCode, ok := response["code"]; apiError.HttpStatusCode == 0 && ok {
+			if statusCodeFloat, ok := statusCode.(float64); ok {
+				apiError.HttpStatusCode = int(statusCodeFloat)
+			}
+		}
+		if response["message"] != nil {
+			errMsg := response["message"].(string)
+			apiError.InternalMessage = errMsg
+			apiError.UserMessage = errMsg
+		}
+	}
+	return apiError
+}
+
 func (impl ArgoK8sClientImpl) GetArgoApplication(namespace string, appName string, cluster *repository.Cluster) (map[string]interface{}, error) {
 
 	config, err := rest.InClusterConfig()

diff --git a/go.mod b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v1.8.0
 	github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1
-	github.com/devtron-labs/common-lib v0.0.23
+	github.com/devtron-labs/common-lib v0.0.24
 	github.com/devtron-labs/go-bitbucket v0.9.60-beta
 	github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1
 	github.com/evanphx/json-patch v5.7.0+incompatible

diff --git a/go.sum b/go.sum
@@ -197,8 +197,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1 h1:qdkpTAo2Kr0ZicZIVXfNwsGSshpc9OB9j9RzmKYdIwY=
 github.com/devtron-labs/authenticator v0.4.35-0.20240607135426-c86e868ecee1/go.mod h1:IkKPPEfgLCMR29he5yv2OCC6iM2R7K5/0AA3k8b9XNc=
-github.com/devtron-labs/common-lib v0.0.23 h1:3n9VgJALanjHrb/t0nSaLbLJcJa7sdsdyFf2leE+cN0=
-github.com/devtron-labs/common-lib v0.0.23/go.mod h1:UZGPt1ep9Tnd9Ak2sibGSiLr7p3ijO2/JLT+h+pqBuU=
+github.com/devtron-labs/common-lib v0.0.24 h1:9Tb1diOnyFFrKJ+RSdzvnBUIQsUhPVaOcyQemU5q2oo=
+github.com/devtron-labs/common-lib v0.0.24/go.mod h1:UZGPt1ep9Tnd9Ak2sibGSiLr7p3ijO2/JLT+h+pqBuU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20240527113333-08a3be5ec6c1 h1:R6qVeFaayqstBSu4w+ipWQqJyMKDqBVV3a11qoA2IaM=

diff --git a/internal/errors/bean.go b/internal/errors/bean.go
@@ -42,6 +42,10 @@ func (r *ClientStatusCode) IsDeadlineExceededCode() bool {
 	return r.Code == codes.DeadlineExceeded
 }
 
+func (r *ClientStatusCode) IsUnavailableCode() bool {
+	return r.Code == codes.Unavailable
+}
+
 func (r *ClientStatusCode) IsCanceledCode() bool {
 	return r.Code == codes.Canceled
 }

diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go
@@ -100,10 +100,6 @@ var currentRequestGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
 	Help: "no of request being served currently",
 }, []string{"path", "method"})
 
-var AcdGetResourceCounter = promauto.NewCounterVec(prometheus.CounterOpts{
-	Name: "acd_get_resource_counter",
-}, []string{"appId", "envId", "acdAppName"})
-
 var CdTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "cd_trigger_counter",
 }, []string{"appName", "envName"})

diff --git a/pkg/clusterTerminalAccess/UserTerminalAccessService.go b/pkg/clusterTerminalAccess/UserTerminalAccessService.go
@@ -366,13 +366,6 @@ func (impl *UserTerminalAccessServiceImpl) DisconnectTerminalSession(ctx context
 	return err
 }
 
-func getErrorDetailedMessage(err error) string {
-	if errStatus, ok := err.(*k8sErrors.StatusError); ok {
-		return errStatus.Status().Message
-	}
-	return ""
-}
-
 func (impl *UserTerminalAccessServiceImpl) StopTerminalSession(ctx context.Context, userTerminalAccessId int) {
 	impl.Logger.Infow("terminal stop request received for user", "userTerminalAccessId", userTerminalAccessId)
 	impl.TerminalAccessDataArrayMutex.Lock()
@@ -529,7 +522,7 @@ func (impl *UserTerminalAccessServiceImpl) SyncPodStatus() {
 			err = impl.DeleteTerminalPod(context.Background(), terminalAccessData.ClusterId, terminalAccessData.PodName, namespace)
 			if err != nil {
 				if k8s.IsResourceNotFoundErr(err) {
-					errorDetailedMessage := getErrorDetailedMessage(err)
+					errorDetailedMessage := k8s.GetClientErrorMessage(err)
 					terminalPodStatusString = fmt.Sprintf("%s/%s", string(models.TerminalPodTerminated), errorDetailedMessage)
 				} else {
 					continue
@@ -841,7 +834,7 @@ func (impl *UserTerminalAccessServiceImpl) getPodManifest(ctx context.Context, c
 	response, err := impl.K8sCommonService.GetResource(ctx, request)
 	if err != nil {
 		if k8s.IsResourceNotFoundErr(err) {
-			errorDetailedMessage := getErrorDetailedMessage(err)
+			errorDetailedMessage := k8s.GetClientErrorMessage(err)
 			terminalPodStatusString := fmt.Sprintf("%s/%s", string(models.TerminalPodTerminated), errorDetailedMessage)
 			return nil, errors.New(terminalPodStatusString)
 		} else {