CIV-0000 Add retry on complete task failure for camunda (#2954)

hmcts · Jul 13, 2023 · 393f51d · 393f51d
1 parent d779a46
commit 393f51d
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 17 deletions.
diff --git a/build.gradle b/build.gradle
@@ -277,7 +277,7 @@ sonarqube {
     property "sonar.projectName", "CIVIL :: service"
     property "sonar.projectKey", "civil-service"
     property "sonar.coverage.jacoco.xmlReportPaths", "${jacocoTestReport.reports.xml.destination.path}"
-    property "sonar.coverage.exclusions", "**/model/**, **/config/**/*Configuration.java, **/testingsupport/**, **/*ExternalTaskListener.java, **/stereotypes/**, **/*Exception.java, **/EventHistoryMapper*.java, **/model/hearingvalues/**, **/enums/hearing/**, **/fees/client/**, **/enums/sdo/**"
+    property "sonar.coverage.exclusions", "**/model/**, **/config/**/*Configuration.java, **/testingsupport/**, **/*ExternalTaskListener.java, **/*BaseExternalTaskHandler.java, **/stereotypes/**, **/*Exception.java, **/EventHistoryMapper*.java, **/model/hearingvalues/**, **/enums/hearing/**, **/fees/client/**, **/enums/sdo/**"
     property "sonar.cpd.exclusions", "**/*DocumentManagementService.java, **/*Spec*.java"
     property "sonar.exclusions", "**/hmc/model/**, **/model/hearingvalues/**"
     property "sonar.host.url", "https://sonar.reform.hmcts.net/"

diff --git a/src/main/java/uk/gov/hmcts/reform/civil/config/ExternalTaskListenerConfiguration.java b/src/main/java/uk/gov/hmcts/reform/civil/config/ExternalTaskListenerConfiguration.java
@@ -8,10 +8,12 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.retry.annotation.EnableRetry;
 import uk.gov.hmcts.reform.authorisation.filters.ServiceAuthFilter;
 import uk.gov.hmcts.reform.authorisation.generators.AuthTokenGenerator;
 
 @Configuration
+@EnableRetry
 public class ExternalTaskListenerConfiguration {
 
     private final String baseUrl;

diff --git a/src/main/java/uk/gov/hmcts/reform/civil/exceptions/CompleteTaskException.java b/src/main/java/uk/gov/hmcts/reform/civil/exceptions/CompleteTaskException.java
@@ -1,6 +1,6 @@
 package uk.gov.hmcts.reform.civil.exceptions;
 
-public class CompleteTaskException extends Exception {
+public class CompleteTaskException extends RuntimeException {
 
     public CompleteTaskException(Throwable cause) {
         super(cause);

diff --git a/src/main/java/uk/gov/hmcts/reform/civil/handler/tasks/BaseExternalTaskHandler.java b/src/main/java/uk/gov/hmcts/reform/civil/handler/tasks/BaseExternalTaskHandler.java
@@ -1,7 +1,5 @@
 package uk.gov.hmcts.reform.civil.handler.tasks;
 
-import java.util.Arrays;
-
 import feign.FeignException;
 import org.camunda.bpm.client.task.ExternalTask;
 import org.camunda.bpm.client.task.ExternalTaskHandler;
@@ -10,9 +8,14 @@
 import org.camunda.bpm.engine.variable.VariableMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.retry.annotation.Backoff;
+import org.springframework.retry.annotation.Recover;
+import org.springframework.retry.annotation.Retryable;
 import uk.gov.hmcts.reform.civil.exceptions.CompleteTaskException;
 import uk.gov.hmcts.reform.civil.exceptions.NotRetryableException;
 
+import java.util.Arrays;
+
 import static java.util.Optional.ofNullable;
 import static uk.gov.hmcts.reform.civil.helpers.ExponentialRetryTimeoutHelper.calculateExponentialRetryTimeout;
 
@@ -39,7 +42,8 @@ default void execute(ExternalTask externalTask, ExternalTaskService externalTask
 
         try {
             log.info("External task '{}' started with processInstanceId '{}'",
-                     topicName, processInstanceId);
+                     topicName, processInstanceId
+            );
             handleTask(externalTask);
             completeTask(externalTask, externalTaskService);
         } catch (BpmnError e) {
@@ -49,15 +53,18 @@ default void execute(ExternalTask externalTask, ExternalTaskService externalTask
             externalTaskService.handleBpmnError(externalTask, e.getErrorCode());
         } catch (NotRetryableException e) {
             log.error("External task '{}' errored  with processInstanceId '{}'",
-                      topicName, processInstanceId, e);
+                      topicName, processInstanceId, e
+            );
             handleFailureNoRetryable(externalTask, externalTaskService, e);
         } catch (Exception e) {
             log.error("External task before handleFailure '{}' errored  with processInstanceId '{}'",
-                      topicName, processInstanceId, e);
+                      topicName, processInstanceId, e
+            );
             handleFailure(externalTask, externalTaskService, e);
         }
     }
 
+    @Retryable(value = CompleteTaskException.class, maxAttempts = 5, backoff = @Backoff(delay = 500))
     default void completeTask(ExternalTask externalTask, ExternalTaskService externalTaskService) throws CompleteTaskException {
         String topicName = externalTask.getTopicName();
         String processInstanceId = externalTask.getProcessInstanceId();
@@ -68,14 +75,24 @@ default void completeTask(ExternalTask externalTask, ExternalTaskService externa
                 () -> externalTaskService.complete(externalTask)
             );
             log.info("External task '{}' finished with processInstanceId '{}'",
-                     topicName, processInstanceId);
-        } catch (Exception e) {
+                     topicName, processInstanceId
+            );
+        } catch (Throwable e) {
             log.error("Completing external task '{}' errored  with processInstanceId '{}'",
-                      topicName, processInstanceId, e);
+                      topicName, processInstanceId, e
+            );
             throw new CompleteTaskException(e);
         }
     }
 
+    @Recover
+    default void recover(CompleteTaskException exception, ExternalTask externalTask, ExternalTaskService externalTaskService) {
+        log.error("Recover CompleteTaskException for external task '{}' errored  with processInstanceId '{}'",
+                  externalTask.getTopicName(), externalTask.getProcessInstanceId(), exception
+        );
+        externalTaskService.complete(externalTask);
+    }
+
     /**
      * Called when an exception arises from the {@link BaseExternalTaskHandler handleTask(externalTask)} method.
      *
@@ -86,13 +103,14 @@ default void completeTask(ExternalTask externalTask, ExternalTaskService externa
     default void handleFailure(ExternalTask externalTask, ExternalTaskService externalTaskService, Exception e) {
         int maxRetries = getMaxAttempts();
         int remainingRetries = externalTask.getRetries() == null ? maxRetries : externalTask.getRetries();
-        log.info("Handle failure externalTask.getRetries() is null ?? '{}' processInstanceId: '{}' " +
-                     "remainingRetries value : '{}' externalTask.getRetries() value: '{}' maxRetries: '{}'",
-                 externalTask.getRetries() != null ? false : true,
-                 externalTask.getProcessInstanceId() != null ? externalTask.getProcessInstanceId() : "Instance id is null",
-                 remainingRetries,
-                 externalTask.getRetries(),
-                 maxRetries
+        log.info(
+            "Handle failure externalTask.getRetries() is null ?? '{}' processInstanceId: '{}' " +
+                "remainingRetries value : '{}' externalTask.getRetries() value: '{}' maxRetries: '{}'",
+            externalTask.getRetries() != null ? false : true,
+            externalTask.getProcessInstanceId() != null ? externalTask.getProcessInstanceId() : "Instance id is null",
+            remainingRetries,
+            externalTask.getRetries(),
+            maxRetries
         );
 
         externalTaskService.handleFailure(