#298 Fix 'pramen-extras' requiring 'delta' format for unit tests.

AbsaOSS · Nov 22, 2023 · 383cf07 · 383cf07
1 parent a3a93ae
commit 383cf07
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 9 deletions.
diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala
@@ -120,7 +120,6 @@ object RunnerCommons {
       log.info(s"Fetching '$file'...")
       val fs = new Path(file).getFileSystem(hadoopConfig)
 
-
       fs.copyToLocalFile(new Path(file), currentPath)
     })
   }

diff --git a/pramen/project/Dependencies.scala b/pramen/project/Dependencies.scala
@@ -42,12 +42,15 @@ object Dependencies {
     "org.mockito"          %  "mockito-core"               % mockitoVersion             % Test,
     "de.flapdoodle.embed"  %  "de.flapdoodle.embed.mongo"  % embeddedMongoDbVersion     % Test,
     "org.hsqldb"           %  "hsqldb"                     % hsqlDbVersion              % Test classifier "jdk8"
-  ) :+ getDeltaDependency(sparkVersion(scalaVersion), isDeltaCompile)
+  ) :+ getDeltaDependency(sparkVersion(scalaVersion), isDeltaCompile, isTest = false)
 
   def ExtrasJobsDependencies(scalaVersion: String): Seq[ModuleID] = Seq(
     "org.apache.spark"     %% "spark-sql"                  % sparkVersion(scalaVersion) % Provided,
     "net.sourceforge.jtds" %  "jtds"                       % msSqlDriverVersion,
     "org.scalatest"        %% "scalatest"                  % scalatestVersion           % Test
-  ) :+ getAbrisDependency(scalaVersion)
+  ) ++ Seq(
+    getAbrisDependency(scalaVersion),
+    getDeltaDependency(sparkVersion(scalaVersion), isCompile = false, isTest = true)
+  )
 
 }
diff --git a/pramen/project/Versions.scala b/pramen/project/Versions.scala
@@ -67,7 +67,7 @@ object Versions {
     }
   }
 
-  def getDeltaDependency(sparkVersion: String, isCompile: Boolean): ModuleID = {
+  def getDeltaDependency(sparkVersion: String, isCompile: Boolean, isTest: Boolean): ModuleID = {
     // According to this: https://docs.delta.io/latest/releases.html
     val (deltaArtifact, deltaVersion) = sparkVersion match {
       case version if version.startsWith("2.")   => ("delta-core", "0.6.1")
@@ -79,12 +79,16 @@ object Versions {
       case version if version.startsWith("3.5.") => ("delta-spark", "3.0.0")  // 'delta-core' was renamed to 'delta-spark' since 3.0.0.
       case _                                     => throw new IllegalArgumentException(s"Spark $sparkVersion not supported.")
     }
-    if (isCompile) {
-      println(s"Using Delta version $deltaArtifact:$deltaVersion (compile)")
-      "io.delta" %% deltaArtifact % deltaVersion % Compile
+    if (isTest) {
+      "io.delta" %% deltaArtifact % deltaVersion % Test
     } else {
-      println(s"Using Delta version $deltaArtifact:$deltaVersion (provided)")
-      "io.delta" %% deltaArtifact % deltaVersion % Provided
+      if (isCompile) {
+        println(s"Using Delta version $deltaArtifact:$deltaVersion (compile)")
+        "io.delta" %% deltaArtifact % deltaVersion % Compile
+      } else {
+        println(s"Using Delta version $deltaArtifact:$deltaVersion (provided)")
+        "io.delta" %% deltaArtifact % deltaVersion % Provided
+      }
     }
   }
-Original file line number
+Diff line change
@@ Expand Up / @@ -120,7 +120,6 @@ object RunnerCommons { @@
           log.info(s"Fetching '$file'...")
           val fs = new Path(file).getFileSystem(hadoopConfig)
           fs.copyToLocalFile(new Path(file), currentPath)
         })
       }
@@ Expand Down @@