Fix unit-tests for Spark-340

Signed-off-by: Ahmed Hussein (amahussein) <[email protected]> Fixes NVIDIA#498, Fixes NVIDIA#463 - Update the gh-workflow to run mvn for list of spark-versions - Fixes the delta-lake dependency in the pom.xml file - Fix the broken unit test for spark-340 - Update the spark-341 to point to release instead of snapshot - Fix the Autotuner warning
amahussein · Aug 22, 2023 · 56db281 · 56db281
1 parent 5ef39b8
commit 56db281
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 8 deletions.
diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -22,7 +22,9 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        spark-version: ['311', '320', '330', '341']
     steps:
     - uses: actions/checkout@v3
 
@@ -32,5 +34,5 @@ jobs:
         distribution: adopt
         java-version: 8
 
-    - name: Run mvn verify
-      run: cd core && mvn verify
+    - name: Run mvn verify with Spark ${{ matrix.spark-version }}
+      run: cd core && mvn -Dbuildver=${{ matrix.spark-version }} verify
diff --git a/core/pom.xml b/core/pom.xml
@@ -327,7 +327,7 @@
         <spark332.version>3.3.2</spark332.version>
         <spark333.version>3.3.3-SNAPSHOT</spark333.version>
         <spark340.version>3.4.0</spark340.version>
-        <spark341.version>3.4.1-SNAPSHOT</spark341.version>
+        <spark341.version>3.4.1</spark341.version>
         <spark350.version>3.5.0-SNAPSHOT</spark350.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
@@ -361,7 +361,7 @@
         <delta21x.version>2.1.1</delta21x.version>
         <delta22x.version>2.2.0</delta22x.version>
         <delta23x.version>2.3.0</delta23x.version>
-        <delta24x.version>2.4.0rc1</delta24x.version>
+        <delta24x.version>2.4.0</delta24x.version>
         <delta.core.version>${delta10x.version}</delta.core.version>
         <java.version>1.8</java.version>
         <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>

diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/AutoTuner.scala
@@ -763,8 +763,6 @@ class AutoTuner(
           && appInfoProvider.getRedundantReadSize > DEF_READ_SIZE_THRESHOLD) {
       appendRecommendation("spark.rapids.filecache.enabled", "true")
       appendComment("Enable file cache only if Spark local disks bandwidth is > 1 GB/s")
-    } else {
-      null
     }
   }
 

diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala
@@ -1154,7 +1154,8 @@ class QualificationSuite extends BaseTestSuite {
         try {
           val lines = inputSource.getLines.toSeq
           // 1 for header, 1 for values
-          assert(lines.size == 6)
+          val expLinesSize = if (ToolUtils.isSpark340OrLater()) 8 else 6
+          assert(lines.size == expLinesSize)
           assert(lines.head.contains("App ID,Unsupported Type,"))
           assert(lines(1).contains("\"Read\",\"JSON\",\"Types not supported - bigint:int\""))
         } finally {