#488 Make Pramen more strict on successes and failures of pipelines.

AbsaOSS · Jan 9, 2025 · 1b42261 · 1b42261
1 parent b031c17
commit 1b42261
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 5 deletions.
diff --git a/pramen/core/src/main/resources/reference.conf b/pramen/core/src/main/resources/reference.conf
@@ -187,6 +187,10 @@ pramen {
     # The maximum length of errors and exceptions in the notification body. The default value is selected
     # so that Pramen can handle at least 100 exceptions in a single email notification.
     exception.max.length = 65536
+
+    # If true, possible statuses are: Failed, Succeeded, and Succeeded with Warnings. "Partial success" is considered a failure.
+    # If false, possible statuses are: Failed, Paraially suceeded, Succeeded, and Succeeded with Warnings
+    strict.failures = true
   }
 }
 

diff --git a/...c/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala b/...c/main/scala/za/co/absa/pramen/core/notify/pipeline/PipelineNotificationBuilderHtml.scala
@@ -39,6 +39,7 @@ object PipelineNotificationBuilderHtml {
   val MIN_MEGABYTES = 10
   val NOTIFICATION_REASON_MAX_LENGTH_KEY = "pramen.notifications.reason.max.length"
   val NOTIFICATION_EXCEPTION_MAX_LENGTH_KEY = "pramen.notifications.exception.max.length"
+  val NOTIFICATION_STRICT_FAILURES_KEY = "pramen.notifications.strict.failures"
   val SUPPRESS_WARNING_STARTING_WITH = "Based on outdated tables: "
 }
 
@@ -58,6 +59,7 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
 
   private val maxReasonLength = ConfigUtils.getOptionInt(conf, NOTIFICATION_REASON_MAX_LENGTH_KEY)
   private val maxExceptionLength = ConfigUtils.getOptionInt(conf, NOTIFICATION_EXCEPTION_MAX_LENGTH_KEY)
+  private val strictFailures = ConfigUtils.getOptionBoolean(conf, NOTIFICATION_STRICT_FAILURES_KEY).getOrElse(true)
 
   var appException: Option[Throwable] = None
   var warningFlag: Boolean = false
@@ -182,15 +184,16 @@ class PipelineNotificationBuilderHtml(implicit conf: Config) extends PipelineNot
   def pipelineStatus: PipelineStatus = {
     val isCertainFailure = appException.nonEmpty
     val (someTasksSucceeded, someTasksFailed) = getSuccessFlags
-    val hasAtLeastOneWarning = warningFlag || hasWarnings
+    val hasInvalidEmails = validatedEmailsOpt.exists(v => v.invalidDomainEmails.nonEmpty || v.invalidFormatEmails.nonEmpty)
+    val hasAtLeastOneWarning = warningFlag || hasWarnings || hasInvalidEmails
 
     if (isCertainFailure) {
       PipelineStatus.Failure
     } else if (!someTasksFailed && !hasAtLeastOneWarning) {
       PipelineStatus.Success
-    } else if (someTasksSucceeded && someTasksFailed) {
+    } else if (someTasksSucceeded && someTasksFailed && !strictFailures) {
       PipelineStatus.PartialSuccess
-    } else if (someTasksSucceeded && hasAtLeastOneWarning) {
+    } else if (someTasksSucceeded && !someTasksFailed && hasAtLeastOneWarning) {
       PipelineStatus.Warning
     } else {
       PipelineStatus.Failure

diff --git a/...a/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala b/...a/za/co/absa/pramen/core/tests/notify/pipeline/PipelineNotificationBuilderHtmlSuite.scala
@@ -94,6 +94,16 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
 
       assert(builder.renderSubject().startsWith("Notification of PARTIAL SUCCESS for MyNewApp at"))
     }
+
+    "render failure when strict failures are turned on" in {
+      val builder = getBuilder(structFailure = true)
+
+      builder.addAppName("MyNewApp")
+      builder.addCompletedTask(TaskResultFactory.getDummyTaskResult(runStatus = TestPrototypes.runStatusWarning))
+      builder.addCompletedTask(TaskResultFactory.getDummyTaskResult(runStatus = TestPrototypes.runStatusFailure))
+
+      assert(builder.renderSubject().startsWith("Notification of FAILURE for MyNewApp at"))
+    }
   }
 
   "renderBody()" should {
@@ -673,13 +683,16 @@ class PipelineNotificationBuilderHtmlSuite extends AnyWordSpec with TextComparis
     }
   }
 
-  def getBuilder(conf: Config = emptyConfig): PipelineNotificationBuilderHtml  = {
+  def getBuilder(conf: Config = emptyConfig,
+                 structFailure: Boolean = false): PipelineNotificationBuilderHtml = {
     implicit val implicitConfig: Config =
       conf.withFallback(
         ConfigFactory.parseString(
-          """pramen {
+          s"""pramen {
             |  application.version = 1.0.0
             |  timezone = "Africa/Johannesburg"
+            |
+            |  notifications.strict.failures = $structFailure
             |}
             |""".stripMargin)
       )