From e4cdac22d12c445eab5559898e112cafbe83adf1 Mon Sep 17 00:00:00 2001 From: Velkov Date: Wed, 25 Sep 2019 13:36:08 +0300 Subject: [PATCH] Introduce configurable retry on operation failure --- clients/models/error_type.go | 61 +++++++++++++++++++++++++++ clients/models/operation.go | 23 ++++++++++ clients/swagger/mta_rest.yaml | 7 +++ commands/action.go | 50 +++++++++++++++------- commands/action_test.go | 18 ++++---- commands/base_command.go | 7 +-- commands/base_command_test.go | 8 ++-- commands/blue_green_deploy_command.go | 21 ++++----- commands/deploy_command.go | 10 +++-- commands/execution_monitor.go | 23 +++++++++- commands/execution_monitor_test.go | 26 +++++++----- commands/monitor_action.go | 5 ++- commands/test.mtar | 0 commands/undeploy_command.go | 9 ++-- util/test.mtar | 0 15 files changed, 206 insertions(+), 62 deletions(-) create mode 100644 clients/models/error_type.go create mode 100644 commands/test.mtar create mode 100644 util/test.mtar diff --git a/clients/models/error_type.go b/clients/models/error_type.go new file mode 100644 index 0000000..7cf8a2e --- /dev/null +++ b/clients/models/error_type.go @@ -0,0 +1,61 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "encoding/json" + + strfmt "github.com/go-openapi/strfmt" + + "github.com/go-openapi/errors" + "github.com/go-openapi/validate" +) + +// ErrorType error type +// swagger:model ErrorType +type ErrorType string + +const ( + // ErrorTypeCONTENT captures enum value "CONTENT" + ErrorTypeCONTENT = "CONTENT" + // ErrorTypeINFRASTRUCTURE captures enum value "INFRASTRUCTURE" + ErrorTypeINFRASTRUCTURE = "INFRASTRUCTURE" +) + +// for schema +var errorTypeEnum []interface{} + +func init() { + var res []ErrorType + if err := json.Unmarshal([]byte(`["CONTENT", "INFRASTRCTURE"]`), &res); err != nil { + panic(err) + } + for _, v := range res { + errorTypeEnum = append(errorTypeEnum, v) + } +} + +func (m ErrorType) validateErrorTypeEnum(path, location string, value ErrorType) error { + if err := validate.Enum(path, location, value, errorTypeEnum); err != nil { + return err + } + return nil +} + +// Validate validates this error type +func (m ErrorType) Validate(formats strfmt.Registry) error { + var res []error + + // value enum + if err := m.validateErrorTypeEnum("", "body", m); err != nil { + return err + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/clients/models/operation.go b/clients/models/operation.go index 5fcaacc..3a62cb8 100644 --- a/clients/models/operation.go +++ b/clients/models/operation.go @@ -44,6 +44,9 @@ type Operation struct { // state State State `json:"state,omitempty"` + // error type + ErrorType ErrorType `json:"errorType,omitempty"` + // user User string `json:"user,omitempty"` } @@ -66,6 +69,8 @@ type Operation struct { /* polymorph Operation state false */ +/* polymorph Operation errorType false */ + /* polymorph Operation user false */ // Validate validates this operation @@ -77,6 +82,10 @@ func (m *Operation) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateErrorType(formats); err != nil { + res = append(res, err) + } + if len(res) > 0 { return errors.CompositeValidationError(res...) } @@ -99,6 +108,20 @@ func (m *Operation) validateState(formats strfmt.Registry) error { return nil } +func (m *Operation) validateErrorType(formats strfmt.Registry) error { + if swag.IsZero(m.ErrorType) { + return nil + } + + if err := m.ErrorType.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("errorType") + } + return nil + } + return nil +} + // MarshalBinary interface implementation func (m *Operation) MarshalBinary() ([]byte, error) { if m == nil { diff --git a/clients/swagger/mta_rest.yaml b/clients/swagger/mta_rest.yaml index b748300..056329d 100644 --- a/clients/swagger/mta_rest.yaml +++ b/clients/swagger/mta_rest.yaml @@ -314,6 +314,8 @@ definitions: type: "boolean" state: $ref: "#/definitions/State" + errorType: + $ref: "#/definitions/ErrorType" messages: type: "array" items: @@ -366,6 +368,11 @@ definitions: - "ERROR" - "ABORTED" - "ACTION_REQUIRED" + ErrorType: + type: "string" + enum: + - "CONTENT" + - "INFRASTRUCTURE" MessageType: type: "string" enum: diff --git a/commands/action.go b/commands/action.go index d0b3263..f9b982a 100644 --- a/commands/action.go +++ b/commands/action.go @@ -14,40 +14,55 @@ type Action interface { } // GetActionToExecute returns the action to execute specified with action id -func GetActionToExecute(actionID, commandName string) Action { +func GetActionToExecute(actionID, commandName string, monitoringRetries uint) Action { switch actionID { case "abort": - action := newAction(actionID) + action := newAction(actionID, VerbosityLevelVERBOSE) return &action case "retry": - action := newMonitoringAction(actionID, commandName) + action := newMonitoringAction(actionID, commandName, VerbosityLevelVERBOSE, monitoringRetries) return &action case "resume": - action := newMonitoringAction(actionID, commandName) + action := newMonitoringAction(actionID, commandName, VerbosityLevelVERBOSE, monitoringRetries) return &action case "monitor": return &MonitorAction{ - commandName: commandName, + commandName: commandName, + monitoringRetries: monitoringRetries, } } return nil } -func newMonitoringAction(actionID, commandName string) monitoringAction { +func GetNoRetriesActionToExecute(actionID, commandName string) Action { + return GetActionToExecute(actionID, commandName, 0) +} + +func newMonitoringAction(actionID, commandName string, verbosityLevel VerbosityLevel, monitoringRetries uint) monitoringAction { return monitoringAction{ - action: newAction(actionID), - commandName: commandName, + action: newAction(actionID, verbosityLevel), + commandName: commandName, + monitoringRetries: monitoringRetries, } } -func newAction(actionID string) action { +func newAction(actionID string, verbosityLevel VerbosityLevel) action { return action{ - actionID: actionID, + actionID: actionID, + verbosityLevel: verbosityLevel, } } +type VerbosityLevel int + +const ( + VerbosityLevelVERBOSE VerbosityLevel = 0 + VerbosityLevelSILENT VerbosityLevel = 1 +) + type action struct { - actionID string + actionID string + verbosityLevel VerbosityLevel } func (a *action) Execute(operationID string, mtaClient mtaclient.MtaClientOperations) ExecutionStatus { @@ -64,13 +79,17 @@ func (a *action) executeInSession(operationID string, mtaClient mtaclient.MtaCli return Failure } - ui.Say("Executing action '%s' on operation %s...", a.actionID, terminal.EntityNameColor(operationID)) + if a.verbosityLevel == VerbosityLevelVERBOSE { + ui.Say("Executing action '%s' on operation %s...", a.actionID, terminal.EntityNameColor(operationID)) + } _, err = mtaClient.ExecuteAction(operationID, a.actionID) if err != nil { ui.Failed("Could not execute action '%s' on operation %s: %s", a.actionID, terminal.EntityNameColor(operationID), err) return Failure } - ui.Ok() + if a.verbosityLevel == VerbosityLevelVERBOSE { + ui.Ok() + } return Success } @@ -85,7 +104,8 @@ func (a *action) actionIsPossible(possibleActions []string) bool { type monitoringAction struct { action - commandName string + commandName string + monitoringRetries uint } func (a *monitoringAction) Execute(operationID string, mtaClient mtaclient.MtaClientOperations) ExecutionStatus { @@ -103,7 +123,7 @@ func (a *monitoringAction) Execute(operationID string, mtaClient mtaclient.MtaCl return status } - return NewExecutionMonitor(a.commandName, operationID, "messages", operation.Messages, mtaClient).Monitor() + return NewExecutionMonitor(a.commandName, operationID, "messages", a.monitoringRetries, operation.Messages, mtaClient).Monitor() } func getMonitoringOperation(operationID string, mtaClient mtaclient.MtaClientOperations) (*models.Operation, error) { diff --git a/commands/action_test.go b/commands/action_test.go index 973fe09..0c2ac15 100644 --- a/commands/action_test.go +++ b/commands/action_test.go @@ -31,7 +31,7 @@ var _ = Describe("Actions", func() { const actionID = "abort" Describe("ExecuteAction", func() { BeforeEach(func() { - action = commands.GetActionToExecute(actionID, commandName) + action = commands.GetNoRetriesActionToExecute(actionID, commandName) mtaClient = fakes.NewFakeMtaClientBuilder(). GetOperationActions(operationID, []string{actionID}, nil). ExecuteAction(operationID, actionID, mtaclient.ResponseHeader{}, nil). @@ -87,7 +87,7 @@ var _ = Describe("Actions", func() { const actionID = "retry" Describe("ExecuteAction", func() { BeforeEach(func() { - action = commands.GetActionToExecute(actionID, commandName) + action = commands.GetNoRetriesActionToExecute(actionID, commandName) mtaClient = fakes.NewFakeMtaClientBuilder(). GetOperationActions(operationID, []string{actionID}, nil). ExecuteAction(operationID, actionID, mtaclient.ResponseHeader{Location: "operations/" + operationID + "?embed=messages"}, nil). @@ -123,7 +123,7 @@ var _ = Describe("Actions", func() { const actionID = "monitor" Describe("ExecuteAction", func() { BeforeEach(func() { - action = commands.GetActionToExecute(actionID, commandName) + action = commands.GetNoRetriesActionToExecute(actionID, commandName) }) Context("when the operation finishes successfully", func() { It("should monitor the operation successfully", func() { @@ -162,28 +162,28 @@ var _ = Describe("Actions", func() { }) }) - Describe("GetActionToExecute", func() { + Describe("GetNoRetriesActionToExecute", func() { Context("with correct action id", func() { It("should return abort action to execute", func() { - actionToExecute := commands.GetActionToExecute("abort", "deploy") + actionToExecute := commands.GetNoRetriesActionToExecute("abort", "deploy") Expect(actionToExecute).NotTo(BeNil()) }) It("should return retry action to execute", func() { - actionToExecute := commands.GetActionToExecute("retry", "deploy") + actionToExecute := commands.GetNoRetriesActionToExecute("retry", "deploy") Expect(actionToExecute).NotTo(BeNil()) }) It("should return resume action to execute", func() { - actionToExecute := commands.GetActionToExecute("resume", "deploy") + actionToExecute := commands.GetNoRetriesActionToExecute("resume", "deploy") Expect(actionToExecute).NotTo(BeNil()) }) It("should return monitor action to execute", func() { - actionToExecute := commands.GetActionToExecute("monitor", "deploy") + actionToExecute := commands.GetNoRetriesActionToExecute("monitor", "deploy") Expect(actionToExecute).NotTo(BeNil()) }) }) Context("with incorrect action id", func() { It("should return nil", func() { - actionToExecute := commands.GetActionToExecute("test", "deploy") + actionToExecute := commands.GetNoRetriesActionToExecute("test", "deploy") Expect(actionToExecute).To(BeNil()) }) }) diff --git a/commands/base_command.go b/commands/base_command.go index 70f3f3d..8173ae4 100644 --- a/commands/base_command.go +++ b/commands/base_command.go @@ -40,6 +40,7 @@ const ( noFailOnMissingPermissionsOpt = "do-not-fail-on-missing-permissions" abortOnErrorOpt = "abort-on-error" deployServiceHost = "deploy-service" + retriesOpt = "retries" ) // BaseCommand represents a base command @@ -234,7 +235,7 @@ func (c *BaseCommand) GetCustomDeployServiceURL(args []string) string { } // ExecuteAction executes the action over the process specified with operationID -func (c *BaseCommand) ExecuteAction(operationID, actionID, host string) ExecutionStatus { +func (c *BaseCommand) ExecuteAction(operationID, actionID string, retries uint, host string) ExecutionStatus { // Create REST client mtaClient, err := c.NewMtaClient(host) if err != nil { @@ -255,7 +256,7 @@ func (c *BaseCommand) ExecuteAction(operationID, actionID, host string) Executio } // Finds the action specified with the actionID - action := GetActionToExecute(actionID, c.name) + action := GetActionToExecute(actionID, c.name, retries) if action == nil { ui.Failed("Invalid action %s", terminal.EntityNameColor(actionID)) return Failure @@ -280,7 +281,7 @@ func (c *BaseCommand) CheckOngoingOperation(mtaID string, host string, force boo if ongoingOperation != nil { // Abort the conflict process if confirmed by the user if c.shouldAbortConflictingOperation(mtaID, force) { - action := GetActionToExecute("abort", c.name) + action := GetNoRetriesActionToExecute("abort", c.name) status := action.Execute(ongoingOperation.ProcessID, mtaClient) if status == Failure { return false, nil diff --git a/commands/base_command_test.go b/commands/base_command_test.go index 0c080ac..b1edab4 100644 --- a/commands/base_command_test.go +++ b/commands/base_command_test.go @@ -190,7 +190,7 @@ var _ = Describe("BaseCommand", func() { Context("with valid process id and valid action id", func() { It("should abort and exit with zero status", func() { output, status := oc.CaptureOutputAndStatus(func() int { - return command.ExecuteAction("test-process-id", "abort", "test-host").ToInt() + return command.ExecuteAction("test-process-id", "abort", 0, "test-host").ToInt() }) ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'abort' on operation test-process-id...\n", "OK\n"}) }) @@ -198,7 +198,7 @@ var _ = Describe("BaseCommand", func() { Context("with non-valid process id and valid action id", func() { It("should return error and exit with non-zero status", func() { output, status := oc.CaptureOutputAndStatus(func() int { - return command.ExecuteAction("not-valid-process-id", "abort", "test-host").ToInt() + return command.ExecuteAction("not-valid-process-id", "abort", 0, "test-host").ToInt() }) ex.ExpectFailure(status, output, "Multi-target app operation with id not-valid-process-id not found") }) @@ -207,7 +207,7 @@ var _ = Describe("BaseCommand", func() { Context("with valid process id and invalid action id", func() { It("should return error and exit with non-zero status", func() { output, status := oc.CaptureOutputAndStatus(func() int { - return command.ExecuteAction("test-process-id", "not-existing-action", "test-host").ToInt() + return command.ExecuteAction("test-process-id", "not-existing-action", 0, "test-host").ToInt() }) ex.ExpectFailure(status, output, "Invalid action not-existing-action") }) @@ -216,7 +216,7 @@ var _ = Describe("BaseCommand", func() { Context("with valid process id and valid action id", func() { It("should retry the process and exit with zero status", func() { output, status := oc.CaptureOutputAndStatus(func() int { - return command.ExecuteAction("test-process-id", "retry", "test-host").ToInt() + return command.ExecuteAction("test-process-id", "retry", 0, "test-host").ToInt() }) ex.ExpectSuccessWithOutput(status, output, []string{"Executing action 'retry' on operation test-process-id...\n", "OK\n", "Process finished.\n", "Use \"cf dmol -i test-process-id\" to download the logs of the process.\n"}) diff --git a/commands/blue_green_deploy_command.go b/commands/blue_green_deploy_command.go index 2da3a41..7a26e8d 100644 --- a/commands/blue_green_deploy_command.go +++ b/commands/blue_green_deploy_command.go @@ -27,20 +27,20 @@ func (c *BlueGreenDeployCommand) GetPluginCommand() plugin.Command { HelpText: "Deploy a multi-target app using blue-green deployment", UsageDetails: plugin.Usage{ Usage: `Deploy a multi-target app using blue-green deployment - cf bg-deploy MTA [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u URL] [-f] [--no-start] [--use-namespaces] [--no-namespaces-for-services] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--no-confirm] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--skip-ownership-validation] [--verify-archive-signature] + cf bg-deploy MTA [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u URL] [-f] [--retries RETRIES] [--no-start] [--use-namespaces] [--no-namespaces-for-services] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--no-confirm] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--skip-ownership-validation] [--verify-archive-signature] Perform action on an active deploy operation cf deploy -i OPERATION_ID -a ACTION [-u URL]`, Options: map[string]string{ - extDescriptorsOpt: "Extension descriptors", - deployServiceURLOpt: "Deploy service URL, by default 'deploy-service.'", - timeoutOpt: "Start timeout in seconds", - versionRuleOpt: "Version rule (HIGHER, SAME_HIGHER, ALL)", - operationIDOpt: "Active deploy operation id", - actionOpt: "Action to perform on active deploy operation (abort, retry, monitor)", - forceOpt: "Force deploy without confirmation for aborting conflicting processes", - util.GetShortOption(noStartOpt): "Do not start apps", - util.GetShortOption(useNamespacesOpt): "Use namespaces in app and service names", + extDescriptorsOpt: "Extension descriptors", + deployServiceURLOpt: "Deploy service URL, by default 'deploy-service.'", + timeoutOpt: "Start timeout in seconds", + versionRuleOpt: "Version rule (HIGHER, SAME_HIGHER, ALL)", + operationIDOpt: "Active deploy operation id", + actionOpt: "Action to perform on active deploy operation (abort, retry, monitor)", + forceOpt: "Force deploy without confirmation for aborting conflicting processes", + util.GetShortOption(noStartOpt): "Do not start apps", + util.GetShortOption(useNamespacesOpt): "Use namespaces in app and service names", util.GetShortOption(noNamespacesForServicesOpt): "Do not use namespaces in service names", util.GetShortOption(deleteServicesOpt): "Recreate changed services / delete discontinued services", util.GetShortOption(deleteServiceKeysOpt): "Delete existing service keys and apply the new ones", @@ -52,6 +52,7 @@ func (c *BlueGreenDeployCommand) GetPluginCommand() plugin.Command { util.GetShortOption(abortOnErrorOpt): "Auto-abort the process on any errors", util.GetShortOption(skipOwnershipValidationOpt): "Skip the ownership validation that prevents the modification of entities managed by other multi-target apps", util.GetShortOption(verifyArchiveSignatureOpt): "Verify the archive is correctly signed", + util.GetShortOption(retriesOpt): "Retry the operation N times in case a non-content error occurs (default 3)", }, }, } diff --git a/commands/deploy_command.go b/commands/deploy_command.go index 434af2a..927307f 100644 --- a/commands/deploy_command.go +++ b/commands/deploy_command.go @@ -80,7 +80,7 @@ func (c *DeployCommand) GetPluginCommand() plugin.Command { HelpText: "Deploy a new multi-target app or sync changes to an existing one", UsageDetails: plugin.Usage{ Usage: `Deploy a multi-target app archive - cf deploy MTA [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u URL] [-f] [--no-start] [--use-namespaces] [--no-namespaces-for-services] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--skip-ownership-validation] [--verify-archive-signature] + cf deploy MTA [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u URL] [-f] [--retries RETRIES] [--no-start] [--use-namespaces] [--no-namespaces-for-services] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--skip-ownership-validation] [--verify-archive-signature] Perform action on an active deploy operation cf deploy -i OPERATION_ID -a ACTION [-u URL]`, @@ -108,6 +108,7 @@ func (c *DeployCommand) GetPluginCommand() plugin.Command { util.GetShortOption(allModulesOpt): "Deploy all modules which are contained in the deployment descriptor, in the current location", util.GetShortOption(allResourcesOpt): "Deploy all resources which are contained in the deployment descriptor, in the current location", util.GetShortOption(verifyArchiveSignatureOpt): "Verify the archive is correctly signed", + util.GetShortOption(retriesOpt): "Retry the operation N times in case a non-content error occurs (default 3)", }, }, } @@ -141,6 +142,7 @@ func deployCommandFlagsDefiner() CommandFlagsDefiner { optionValues[allModulesOpt] = flags.Bool(allModulesOpt, false, "") optionValues[allResourcesOpt] = flags.Bool(allResourcesOpt, false, "") optionValues[verifyArchiveSignatureOpt] = flags.Bool(verifyArchiveSignatureOpt, false, "") + optionValues[retriesOpt] = flags.Uint(retriesOpt, 3, "") flags.Var(&modulesList, moduleOpt, "") flags.Var(&resourcesList, resourceOpt, "") return optionValues @@ -210,6 +212,7 @@ func (c *DeployCommand) Execute(args []string) ExecutionStatus { operationID := GetStringOpt(operationIDOpt, optionValues) action := GetStringOpt(actionOpt, optionValues) force := GetBoolOpt(forceOpt, optionValues) + retries := GetUintOpt(retriesOpt, optionValues) context, err := c.GetContext() if err != nil { @@ -218,7 +221,7 @@ func (c *DeployCommand) Execute(args []string) ExecutionStatus { } if operationID != "" || action != "" { - return c.ExecuteAction(operationID, action, host) + return c.ExecuteAction(operationID, action, retries, host) } mtaElementsCalculator := mtaElementsToAddCalculator{shouldAddAllModules: false, shouldAddAllResources: false} mtaElementsCalculator.calculateElementsToDeploy(optionValues) @@ -321,7 +324,8 @@ func (c *DeployCommand) Execute(args []string) ExecutionStatus { ui.Failed("Could not create operation: %s", baseclient.NewClientError(err)) return Failure } - return NewExecutionMonitorFromLocationHeader(c.name, responseHeader.Location.String(), []*models.Message{}, mtaClient).Monitor() + + return NewExecutionMonitorFromLocationHeader(c.name, responseHeader.Location.String(), retries, []*models.Message{}, mtaClient).Monitor() } func setModulesAndResourcesListParameters(modulesList, resourcesList listFlag, processBuilder *util.ProcessBuilder, mtaElementsCalculator mtaElementsToAddCalculator) { diff --git a/commands/execution_monitor.go b/commands/execution_monitor.go index 5aac02f..bcf9a7e 100644 --- a/commands/execution_monitor.go +++ b/commands/execution_monitor.go @@ -23,9 +23,10 @@ type ExecutionMonitor struct { monitoringLocation string operationID string embed string + retries uint } -func NewExecutionMonitorFromLocationHeader(commandName, location string, reportedOperationMessages []*models.Message, mtaClient mtaclient.MtaClientOperations) *ExecutionMonitor { +func NewExecutionMonitorFromLocationHeader(commandName, location string, retries uint, reportedOperationMessages []*models.Message, mtaClient mtaclient.MtaClientOperations) *ExecutionMonitor { operationID, embed := getMonitoringInformation(location) return &ExecutionMonitor{ mtaClient: mtaClient, @@ -33,6 +34,7 @@ func NewExecutionMonitorFromLocationHeader(commandName, location string, reporte commandName: commandName, operationID: operationID, embed: embed, + retries: retries, } } @@ -44,13 +46,14 @@ func getMonitoringInformation(monitoringLocation string) (string, string) { } //NewExecutionMonitor creates a new execution monitor -func NewExecutionMonitor(commandName, operationID, embed string, reportedOperationMessages []*models.Message, mtaClient mtaclient.MtaClientOperations) *ExecutionMonitor { +func NewExecutionMonitor(commandName, operationID, embed string, retries uint, reportedOperationMessages []*models.Message, mtaClient mtaclient.MtaClientOperations) *ExecutionMonitor { return &ExecutionMonitor{ mtaClient: mtaClient, reportedMessages: getAlreadyReportedOperationMessages(reportedOperationMessages), commandName: commandName, operationID: operationID, embed: embed, + retries: retries, } } @@ -63,6 +66,7 @@ func getAlreadyReportedOperationMessages(reportedOperationMessages []*models.Mes } func (m *ExecutionMonitor) Monitor() ExecutionStatus { + totalRetries := m.retries for { operation, err := m.mtaClient.GetMtaOperation(m.operationID, m.embed) if err != nil { @@ -82,6 +86,11 @@ func (m *ExecutionMonitor) Monitor() ExecutionStatus { m.reportCommandForDownloadOfProcessLogs(m.operationID) return Failure case models.StateERROR: + if canRetry(m.retries, operation) { + ui.Say("Proceeding with automatic retry... (%d of %d attempts left)", m.retries, totalRetries) + executeRetryAction(m) + continue + } messageInError := findErrorMessage(operation.Messages) if messageInError == nil { ui.Failed("There is no error message for operation with id %s", m.operationID) @@ -103,6 +112,16 @@ func (m *ExecutionMonitor) Monitor() ExecutionStatus { } } +func canRetry(retries uint, operation *models.Operation) bool { + return retries > 0 && operation.ErrorType != models.ErrorTypeCONTENT +} + +func executeRetryAction(executionMonitor *ExecutionMonitor) { + retryAction := newAction("retry", VerbosityLevelSILENT) + retryAction.Execute(executionMonitor.operationID, executionMonitor.mtaClient) + executionMonitor.retries-- +} + func findErrorMessage(messages models.OperationMessages) *models.Message { for _, message := range messages { if message.Type == models.MessageTypeERROR { diff --git a/commands/execution_monitor_test.go b/commands/execution_monitor_test.go index 2282f40..ebe6cba 100644 --- a/commands/execution_monitor_test.go +++ b/commands/execution_monitor_test.go @@ -64,7 +64,7 @@ var _ = Describe("ExecutionMonitor", func() { State: "ABORTED", Messages: []*models.Message{}, }, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) @@ -72,8 +72,8 @@ var _ = Describe("ExecutionMonitor", func() { ex.ExpectMessageOnLine(output, "Process was aborted.", 0) }) }) - Context("with process task in state error and no progress messages in the tasklist", func() { - It("should return error and exit with non-zero status", func() { + Context("with process task in state error and no progress messages in the tasklist, retrying 4 times", func() { + It("should retry 4 times, return error and exit with non-zero status", func() { client = fakeMtaClientBuilder. GetMtaOperation(processID, "messages", &models.Operation{ ProcessID: processID, @@ -85,12 +85,16 @@ var _ = Describe("ExecutionMonitor", func() { }, }, }, nil). - GetOperationActions(processID, []string{"abort"}, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + GetOperationActions(processID, []string{"abort", "retry"}, nil).Build() + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 4, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) - ex.ExpectFailureOnLine(status, output, "Use \"cf deploy -i 1234 -a abort\" to abort the process.\n", 1) + ex.ExpectMessageOnLine(output, "Proceeding with automatic retry... (4 of 4 attempts left)", 1) + ex.ExpectMessageOnLine(output, "Proceeding with automatic retry... (3 of 4 attempts left)", 2) + ex.ExpectMessageOnLine(output, "Proceeding with automatic retry... (2 of 4 attempts left)", 3) + ex.ExpectMessageOnLine(output, "Proceeding with automatic retry... (1 of 4 attempts left)", 4) + ex.ExpectFailureOnLine(status, output, "Use \"cf deploy -i 1234 -a abort\" to abort the process.\n", 5) }) }) Context("with process task in illegal state and no progress messages in the tasklist", func() { @@ -100,7 +104,7 @@ var _ = Describe("ExecutionMonitor", func() { State: "UnknownState", Messages: []*models.Message{}, }, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) @@ -115,7 +119,7 @@ var _ = Describe("ExecutionMonitor", func() { State: "FINISHED", Messages: []*models.Message{}, }, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) @@ -135,7 +139,7 @@ var _ = Describe("ExecutionMonitor", func() { testutil.GetMessage(31, "test-message-3"), }, }, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) @@ -155,7 +159,7 @@ var _ = Describe("ExecutionMonitor", func() { testutil.GetMessage(4, "test-message-4"), }, }, nil).Build() - monitor = commands.NewExecutionMonitor(commandName, processID, "messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitor(commandName, processID, "messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) @@ -171,7 +175,7 @@ var _ = Describe("ExecutionMonitor", func() { Messages: []*models.Message{}, }, nil). GetOperationActions(processID, []string{"retry", "abort"}, nil).Build() - monitor = commands.NewExecutionMonitorFromLocationHeader(commandName, "operations/"+processID+"?embed=messages", []*models.Message{}, client) + monitor = commands.NewExecutionMonitorFromLocationHeader(commandName, "operations/"+processID+"?embed=messages", 0, []*models.Message{}, client) output, status := oc.CaptureOutputAndStatus(func() int { return monitor.Monitor().ToInt() }) diff --git a/commands/monitor_action.go b/commands/monitor_action.go index b4e0ca9..78d6df3 100644 --- a/commands/monitor_action.go +++ b/commands/monitor_action.go @@ -8,7 +8,8 @@ import ( // MonitorAction monitors process execution type MonitorAction struct { - commandName string + commandName string + monitoringRetries uint } // Execute executes monitor action on process with the specified id @@ -19,5 +20,5 @@ func (a *MonitorAction) Execute(operationID string, mtaClient mtaclient.MtaClien return Failure } - return NewExecutionMonitor(a.commandName, operationID, "messages", operation.Messages, mtaClient).Monitor() + return NewExecutionMonitor(a.commandName, operationID, "messages", a.monitoringRetries, operation.Messages, mtaClient).Monitor() } diff --git a/commands/test.mtar b/commands/test.mtar new file mode 100644 index 0000000..e69de29 diff --git a/commands/undeploy_command.go b/commands/undeploy_command.go index bbe4b62..4cf01fe 100644 --- a/commands/undeploy_command.go +++ b/commands/undeploy_command.go @@ -33,7 +33,7 @@ func (c *UndeployCommand) GetPluginCommand() plugin.Command { HelpText: "Undeploy a multi-target app", UsageDetails: plugin.Usage{ Usage: `Undeploy a multi-target app - cf undeploy MTA_ID [-u URL] [-f] [--delete-services] [--delete-service-brokers] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] + cf undeploy MTA_ID [-u URL] [-f] [--retries RETRIES] [--delete-services] [--delete-service-brokers] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] Perform action on an active undeploy operation cf undeploy -i OPERATION_ID -a ACTION [-u URL]`, @@ -47,6 +47,7 @@ func (c *UndeployCommand) GetPluginCommand() plugin.Command { util.GetShortOption(noRestartSubscribedAppsOpt): "Do not restart subscribed apps, updated during the undeployment", util.GetShortOption(noFailOnMissingPermissionsOpt): "Do not fail on missing permissions for admin operations", util.GetShortOption(abortOnErrorOpt): "Auto-abort the process on any errors", + util.GetShortOption(retriesOpt): "Retry the operation N times in case a non-content error occurs (default 3)", }, }, } @@ -65,6 +66,7 @@ func (c *UndeployCommand) Execute(args []string) ExecutionStatus { var deleteServiceBrokers bool var noFailOnMissingPermissions bool var abortOnError bool + var retries uint flags, err := c.CreateFlags(&host, args) if err != nil { ui.Failed(err.Error()) @@ -78,6 +80,7 @@ func (c *UndeployCommand) Execute(args []string) ExecutionStatus { flags.BoolVar(&deleteServiceBrokers, deleteServiceBrokersOpt, false, "") flags.BoolVar(&noFailOnMissingPermissions, noFailOnMissingPermissionsOpt, false, "") flags.BoolVar(&abortOnError, abortOnErrorOpt, false, "") + flags.UintVar(&retries, retriesOpt, 3, "") parser := NewCommandFlagsParser(flags, NewProcessActionExecutorCommandArgumentsParser([]string{"MTA_ID"}), NewDefaultCommandFlagsValidator(nil)) err = parser.Parse(args) @@ -93,7 +96,7 @@ func (c *UndeployCommand) Execute(args []string) ExecutionStatus { } if operationID != "" || actionID != "" { - return c.ExecuteAction(operationID, actionID, host) + return c.ExecuteAction(operationID, actionID, retries, host) } mtaID := args[0] @@ -155,7 +158,7 @@ func (c *UndeployCommand) Execute(args []string) ExecutionStatus { } // Monitor process execution - return NewExecutionMonitorFromLocationHeader(c.name, responseHeader.Location.String(), []*models.Message{}, mtaClient).Monitor() + return NewExecutionMonitorFromLocationHeader(c.name, responseHeader.Location.String(), retries, []*models.Message{}, mtaClient).Monitor() } type undeployCommandProcessTypeProvider struct{} diff --git a/util/test.mtar b/util/test.mtar new file mode 100644 index 0000000..e69de29