Introduce configurable retry on operation failure

clients/swagger/mta_rest.yaml

@@ -314,6 +314,8 @@ definitions:
         type: "boolean"
       state:
         $ref: "#/definitions/State"
+      errorType:
+        $ref: "#/definitions/ErrorType"
       messages:
         type: "array"
         items:
@@ -366,6 +368,11 @@ definitions:
     - "ERROR"
     - "ABORTED"
     - "ACTION_REQUIRED"
+  ErrorType:
+    type: "string"
+    enum:
+    - "CONTENT"
+    - "INFRASTRUCTURE"
   MessageType:
     type: "string"
     enum:

commands/action.go

@@ -14,40 +14,55 @@ type Action interface {
 }
 
 // GetActionToExecute returns the action to execute specified with action id
-func GetActionToExecute(actionID, commandName string) Action {
+func GetActionToExecute(actionID, commandName string, monitoringRetries uint) Action {
 	switch actionID {
 	case "abort":
-		action := newAction(actionID)
+		action := newAction(actionID, VerbosityLevelVERBOSE)
 		return &action
 	case "retry":
-		action := newMonitoringAction(actionID, commandName)
+		action := newMonitoringAction(actionID, commandName, VerbosityLevelVERBOSE, monitoringRetries)
 		return &action
 	case "resume":
-		action := newMonitoringAction(actionID, commandName)
+		action := newMonitoringAction(actionID, commandName, VerbosityLevelVERBOSE, monitoringRetries)
 		return &action
 	case "monitor":
 		return &MonitorAction{
-			commandName: commandName,
+			commandName:       commandName,
+			monitoringRetries: monitoringRetries,
 		}
 	}
 	return nil
 }
 
-func newMonitoringAction(actionID, commandName string) monitoringAction {
+func GetNoRetriesActionToExecute(actionID, commandName string) Action {
+	return GetActionToExecute(actionID, commandName, 0)
+}
+
+func newMonitoringAction(actionID, commandName string, verbosityLevel VerbosityLevel, monitoringRetries uint) monitoringAction {
 	return monitoringAction{
-		action:      newAction(actionID),
-		commandName: commandName,
+		action:            newAction(actionID, verbosityLevel),
+		commandName:       commandName,
+		monitoringRetries: monitoringRetries,
 	}
 }
 
-func newAction(actionID string) action {
+func newAction(actionID string, verbosityLevel VerbosityLevel) action {
 	return action{
-		actionID: actionID,
+		actionID:       actionID,
+		verbosityLevel: verbosityLevel,
 	}
 }
 
+type VerbosityLevel int
+
+const (
+	VerbosityLevelVERBOSE VerbosityLevel = 0
+	VerbosityLevelSILENT  VerbosityLevel = 1
+)
+
 type action struct {
-	actionID string
+	actionID       string
+	verbosityLevel VerbosityLevel
 }
 
 func (a *action) Execute(operationID string, mtaClient mtaclient.MtaClientOperations) ExecutionStatus {
@@ -64,13 +79,17 @@ func (a *action) executeInSession(operationID string, mtaClient mtaclient.MtaCli
 		return Failure
 	}
 
-	ui.Say("Executing action '%s' on operation %s...", a.actionID, terminal.EntityNameColor(operationID))
+	if a.verbosityLevel == VerbosityLevelVERBOSE {
+		ui.Say("Executing action '%s' on operation %s...", a.actionID, terminal.EntityNameColor(operationID))
+	}
 	_, err = mtaClient.ExecuteAction(operationID, a.actionID)
 	if err != nil {
 		ui.Failed("Could not execute action '%s' on operation %s: %s", a.actionID, terminal.EntityNameColor(operationID), err)
 		return Failure
 	}
-	ui.Ok()
+	if a.verbosityLevel == VerbosityLevelVERBOSE {
+		ui.Ok()
+	}
 	return Success
 }
 
@@ -85,7 +104,8 @@ func (a *action) actionIsPossible(possibleActions []string) bool {
 
 type monitoringAction struct {
 	action
-	commandName string
+	commandName       string
+	monitoringRetries uint
 }
 
 func (a *monitoringAction) Execute(operationID string, mtaClient mtaclient.MtaClientOperations) ExecutionStatus {
@@ -103,7 +123,7 @@ func (a *monitoringAction) Execute(operationID string, mtaClient mtaclient.MtaCl
 		return status
 	}
 
-	return NewExecutionMonitor(a.commandName, operationID, "messages", operation.Messages, mtaClient).Monitor()
+	return NewExecutionMonitor(a.commandName, operationID, "messages", a.monitoringRetries, operation.Messages, mtaClient).Monitor()
 }
 
 func getMonitoringOperation(operationID string, mtaClient mtaclient.MtaClientOperations) (*models.Operation, error) {

commands/action_test.go

@@ -31,7 +31,7 @@ var _ = Describe("Actions", func() {
 		const actionID = "abort"
 		Describe("ExecuteAction", func() {
 			BeforeEach(func() {
-				action = commands.GetActionToExecute(actionID, commandName)
+				action = commands.GetNoRetriesActionToExecute(actionID, commandName)
 				mtaClient = fakes.NewFakeMtaClientBuilder().
 					GetOperationActions(operationID, []string{actionID}, nil).
 					ExecuteAction(operationID, actionID, mtaclient.ResponseHeader{}, nil).
@@ -87,7 +87,7 @@ var _ = Describe("Actions", func() {
 		const actionID = "retry"
 		Describe("ExecuteAction", func() {
 			BeforeEach(func() {
-				action = commands.GetActionToExecute(actionID, commandName)
+				action = commands.GetNoRetriesActionToExecute(actionID, commandName)
 				mtaClient = fakes.NewFakeMtaClientBuilder().
 					GetOperationActions(operationID, []string{actionID}, nil).
 					ExecuteAction(operationID, actionID, mtaclient.ResponseHeader{Location: "operations/" + operationID + "?embed=messages"}, nil).
@@ -123,7 +123,7 @@ var _ = Describe("Actions", func() {
 		const actionID = "monitor"
 		Describe("ExecuteAction", func() {
 			BeforeEach(func() {
-				action = commands.GetActionToExecute(actionID, commandName)
+				action = commands.GetNoRetriesActionToExecute(actionID, commandName)
 			})
 			Context("when the operation finishes successfully", func() {
 				It("should monitor the operation successfully", func() {
@@ -162,28 +162,28 @@ var _ = Describe("Actions", func() {
 		})
 	})
 
-	Describe("GetActionToExecute", func() {
+	Describe("GetNoRetriesActionToExecute", func() {
 		Context("with correct action id", func() {
 			It("should return abort action to execute", func() {
-				actionToExecute := commands.GetActionToExecute("abort", "deploy")
+				actionToExecute := commands.GetNoRetriesActionToExecute("abort", "deploy")
 				Expect(actionToExecute).NotTo(BeNil())
 			})
 			It("should return retry action to execute", func() {
-				actionToExecute := commands.GetActionToExecute("retry", "deploy")
+				actionToExecute := commands.GetNoRetriesActionToExecute("retry", "deploy")
 				Expect(actionToExecute).NotTo(BeNil())
 			})
 			It("should return resume action to execute", func() {
-				actionToExecute := commands.GetActionToExecute("resume", "deploy")
+				actionToExecute := commands.GetNoRetriesActionToExecute("resume", "deploy")
 				Expect(actionToExecute).NotTo(BeNil())
 			})
 			It("should return monitor action to execute", func() {
-				actionToExecute := commands.GetActionToExecute("monitor", "deploy")
+				actionToExecute := commands.GetNoRetriesActionToExecute("monitor", "deploy")
 				Expect(actionToExecute).NotTo(BeNil())
 			})
 		})
 		Context("with incorrect action id", func() {
 			It("should return nil", func() {
-				actionToExecute := commands.GetActionToExecute("test", "deploy")
+				actionToExecute := commands.GetNoRetriesActionToExecute("test", "deploy")
 				Expect(actionToExecute).To(BeNil())
 			})
 		})

commands/base_command.go

@@ -40,6 +40,7 @@ const (
 	noFailOnMissingPermissionsOpt = "do-not-fail-on-missing-permissions"
 	abortOnErrorOpt               = "abort-on-error"
 	deployServiceHost             = "deploy-service"
+	retriesOpt                    = "retries"
 )
 
 // BaseCommand represents a base command
@@ -234,7 +235,7 @@ func (c *BaseCommand) GetCustomDeployServiceURL(args []string) string {
 }
 
 // ExecuteAction executes the action over the process specified with operationID
-func (c *BaseCommand) ExecuteAction(operationID, actionID, host string) ExecutionStatus {
+func (c *BaseCommand) ExecuteAction(operationID, actionID string, retries uint, host string) ExecutionStatus {
 	// Create REST client
 	mtaClient, err := c.NewMtaClient(host)
 	if err != nil {
@@ -255,7 +256,7 @@ func (c *BaseCommand) ExecuteAction(operationID, actionID, host string) Executio
 	}
 
 	// Finds the action specified with the actionID
-	action := GetActionToExecute(actionID, c.name)
+	action := GetActionToExecute(actionID, c.name, retries)
 	if action == nil {
 		ui.Failed("Invalid action %s", terminal.EntityNameColor(actionID))
 		return Failure
@@ -280,7 +281,7 @@ func (c *BaseCommand) CheckOngoingOperation(mtaID string, host string, force boo
 	if ongoingOperation != nil {
 		// Abort the conflict process if confirmed by the user
 		if c.shouldAbortConflictingOperation(mtaID, force) {
-			action := GetActionToExecute("abort", c.name)
+			action := GetNoRetriesActionToExecute("abort", c.name)
 			status := action.Execute(ongoingOperation.ProcessID, mtaClient)
 			if status == Failure {
 				return false, nil

commands/base_command_test.go

@@ -190,15 +190,15 @@ var _ = Describe("BaseCommand", func() {
 		Context("with valid process id and valid action id", func() {
 			It("should abort and exit with zero status", func() {
 				output, status := oc.CaptureOutputAndStatus(func() int {
-					return command.ExecuteAction("test-process-id", "abort", "test-host").ToInt()
+					return command.ExecuteAction("test-process-id", "abort", 0, "test-host").ToInt()
 				})
 				ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'abort' on operation test-process-id...\n", "OK\n"})
 			})
 		})
 		Context("with non-valid process id and valid action id", func() {
 			It("should return error and exit with non-zero status", func() {
 				output, status := oc.CaptureOutputAndStatus(func() int {
-					return command.ExecuteAction("not-valid-process-id", "abort", "test-host").ToInt()
+					return command.ExecuteAction("not-valid-process-id", "abort", 0, "test-host").ToInt()
 				})
 				ex.ExpectFailure(status, output, "Multi-target app operation with id not-valid-process-id not found")
 			})
@@ -207,7 +207,7 @@ var _ = Describe("BaseCommand", func() {
 		Context("with valid process id and invalid action id", func() {
 			It("should return error and exit with non-zero status", func() {
 				output, status := oc.CaptureOutputAndStatus(func() int {
-					return command.ExecuteAction("test-process-id", "not-existing-action", "test-host").ToInt()
+					return command.ExecuteAction("test-process-id", "not-existing-action", 0, "test-host").ToInt()
 				})
 				ex.ExpectFailure(status, output, "Invalid action not-existing-action")
 			})
@@ -216,7 +216,7 @@ var _ = Describe("BaseCommand", func() {
 		Context("with valid process id and valid action id", func() {
 			It("should retry the process and exit with zero status", func() {
 				output, status := oc.CaptureOutputAndStatus(func() int {
-					return command.ExecuteAction("test-process-id", "retry", "test-host").ToInt()
+					return command.ExecuteAction("test-process-id", "retry", 0, "test-host").ToInt()
 				})
 				ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'retry' on operation test-process-id...\n", "OK\n",
 					"Process finished.\n", "Use \"cf dmol -i test-process-id\" to download the logs of the process.\n"})