diff --git a/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala b/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala index 1f1a4995f..a394bcf08 100644 --- a/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala +++ b/pramen/core/src/main/scala/za/co/absa/pramen/core/RunnerCommons.scala @@ -120,7 +120,6 @@ object RunnerCommons { log.info(s"Fetching '$file'...") val fs = new Path(file).getFileSystem(hadoopConfig) - fs.copyToLocalFile(new Path(file), currentPath) }) } diff --git a/pramen/project/Dependencies.scala b/pramen/project/Dependencies.scala index 52de97f08..8e312a039 100644 --- a/pramen/project/Dependencies.scala +++ b/pramen/project/Dependencies.scala @@ -42,12 +42,15 @@ object Dependencies { "org.mockito" % "mockito-core" % mockitoVersion % Test, "de.flapdoodle.embed" % "de.flapdoodle.embed.mongo" % embeddedMongoDbVersion % Test, "org.hsqldb" % "hsqldb" % hsqlDbVersion % Test classifier "jdk8" - ) :+ getDeltaDependency(sparkVersion(scalaVersion), isDeltaCompile) + ) :+ getDeltaDependency(sparkVersion(scalaVersion), isDeltaCompile, isTest = false) def ExtrasJobsDependencies(scalaVersion: String): Seq[ModuleID] = Seq( "org.apache.spark" %% "spark-sql" % sparkVersion(scalaVersion) % Provided, "net.sourceforge.jtds" % "jtds" % msSqlDriverVersion, "org.scalatest" %% "scalatest" % scalatestVersion % Test - ) :+ getAbrisDependency(scalaVersion) + ) ++ Seq( + getAbrisDependency(scalaVersion), + getDeltaDependency(sparkVersion(scalaVersion), isCompile = false, isTest = true) + ) } diff --git a/pramen/project/Versions.scala b/pramen/project/Versions.scala index 0c61566c3..c8954c5c6 100644 --- a/pramen/project/Versions.scala +++ b/pramen/project/Versions.scala @@ -67,7 +67,7 @@ object Versions { } } - def getDeltaDependency(sparkVersion: String, isCompile: Boolean): ModuleID = { + def getDeltaDependency(sparkVersion: String, isCompile: Boolean, isTest: Boolean): ModuleID = { // According to this: https://docs.delta.io/latest/releases.html val (deltaArtifact, deltaVersion) = sparkVersion match { case version if version.startsWith("2.") => ("delta-core", "0.6.1") @@ -79,12 +79,16 @@ object Versions { case version if version.startsWith("3.5.") => ("delta-spark", "3.0.0") // 'delta-core' was renamed to 'delta-spark' since 3.0.0. case _ => throw new IllegalArgumentException(s"Spark $sparkVersion not supported.") } - if (isCompile) { - println(s"Using Delta version $deltaArtifact:$deltaVersion (compile)") - "io.delta" %% deltaArtifact % deltaVersion % Compile + if (isTest) { + "io.delta" %% deltaArtifact % deltaVersion % Test } else { - println(s"Using Delta version $deltaArtifact:$deltaVersion (provided)") - "io.delta" %% deltaArtifact % deltaVersion % Provided + if (isCompile) { + println(s"Using Delta version $deltaArtifact:$deltaVersion (compile)") + "io.delta" %% deltaArtifact % deltaVersion % Compile + } else { + println(s"Using Delta version $deltaArtifact:$deltaVersion (provided)") + "io.delta" %% deltaArtifact % deltaVersion % Provided + } } }