Merge pull request #11306 from NVIDIA/merge-branch-24.08-to-main

Merge branch-24.08 into main [skip ci]

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-24.06
+    - branch-24.08
     types: [closed]
 
 jobs:
@@ -29,13 +29,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: branch-24.06 # force to fetch from latest upstream instead of PR ref
+          ref: branch-24.08 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids
-          HEAD: branch-24.06
-          BASE: branch-24.08
+          HEAD: branch-24.08
+          BASE: branch-24.10
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/blossom-ci.yml

@@ -33,44 +33,50 @@ jobs:
       args: ${{ env.args }}
 
     # This job only runs for pull request comments
-    if: contains( '\
-      abellina,\
-      anfeng,\
-      firestarman,\
-      GaryShen2008,\
-      jlowe,\
-      kuhushukla,\
-      mythrocks,\
-      nartal1,\
-      nvdbaranec,\
-      NvTimLiu,\
-      razajafri,\
-      revans2,\
-      rwlee,\
-      sameerz,\
-      tgravescs,\
-      wbo4958,\
-      wjxiz1992,\
-      sperlingxx,\
-      hyperbolic2346,\
-      gerashegalov,\
-      ttnghia,\
-      nvliyuan,\
-      res-life,\
-      HaoYang670,\
-      NVnavkumar,\
-      amahussein,\
-      mattahrens,\
-      YanxuanLiu,\
-      cindyyuanjiang,\
-      thirtiseven,\
-      winningsix,\
-      viadea,\
-      yinqingh,\
-      parthosa,\
-      liurenjie1024,\
-      binmahone,\
-      ', format('{0},', github.actor)) && github.event.comment.body == 'build'
+    if: |
+      github.event.comment.body == 'build' &&
+      (
+        github.actor == 'abellina' ||
+        github.actor == 'anfeng' ||
+        github.actor == 'firestarman' ||
+        github.actor == 'GaryShen2008' ||
+        github.actor == 'jlowe' ||
+        github.actor == 'kuhushukla' ||
+        github.actor == 'mythrocks' ||
+        github.actor == 'nartal1' ||
+        github.actor == 'nvdbaranec' ||
+        github.actor == 'NvTimLiu' ||
+        github.actor == 'razajafri' ||
+        github.actor == 'revans2' ||
+        github.actor == 'rwlee' ||
+        github.actor == 'sameerz' ||
+        github.actor == 'tgravescs' ||
+        github.actor == 'wbo4958' ||
+        github.actor == 'wjxiz1992' ||
+        github.actor == 'sperlingxx' ||
+        github.actor == 'hyperbolic2346' ||
+        github.actor == 'gerashegalov' ||
+        github.actor == 'ttnghia' ||
+        github.actor == 'nvliyuan' ||
+        github.actor == 'res-life' ||
+        github.actor == 'HaoYang670' ||
+        github.actor == 'NVnavkumar' ||
+        github.actor == 'amahussein' ||
+        github.actor == 'mattahrens' ||
+        github.actor == 'YanxuanLiu' ||
+        github.actor == 'cindyyuanjiang' ||
+        github.actor == 'thirtiseven' ||
+        github.actor == 'winningsix' ||
+        github.actor == 'viadea' ||
+        github.actor == 'yinqingh' ||
+        github.actor == 'parthosa' ||
+        github.actor == 'liurenjie1024' ||
+        github.actor == 'binmahone' ||
+        github.actor == 'zpuller' ||
+        github.actor == 'pxLi'  ||
+        github.actor == 'Feng-Jiang28' ||
+        github.actor == 'SurajAralihalli'
+      )
     steps:
       - name: Check if comment is issued by authorized person
         run: blossom-ci
@@ -85,15 +91,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
           ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
           lfs: 'true'
 
       # repo specific steps
       - name: Setup java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: adopt
           java-version: 8

.github/workflows/mvn-verify-check.yml

@@ -40,10 +40,9 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       dailyCacheKey: ${{ steps.generateCacheKey.outputs.dailyCacheKey }}
-      defaultSparkVersion: ${{ steps.allShimVersionsStep.outputs.defaultSparkVersion }}
-      sparkTailVersions: ${{ steps.allShimVersionsStep.outputs.tailVersions }}
-      sparkJDKVersions: ${{ steps.allShimVersionsStep.outputs.jdkVersions }}
-      scala213Versions: ${{ steps.allShimVersionsStep.outputs.scala213Versions }}
+      defaultSparkVersion: ${{ steps.all212ShimVersionsStep.outputs.defaultSparkVersion }}
+      sparkTailVersions: ${{ steps.all212ShimVersionsStep.outputs.tailVersions }}
+      sparkJDKVersions: ${{ steps.all212ShimVersionsStep.outputs.jdkVersions }}
     steps:
       - uses: actions/checkout@v4 # refs/pull/:prNumber/merge
       - uses: actions/setup-java@v4
@@ -65,31 +64,12 @@ jobs:
           restore-keys: ${{ runner.os }}-maven-
       - name: populate-daily-cache
         if: steps.cache.outputs.cache-hit != 'true'
+        env:
+          SCALA_VER: '2.12'
         run: |
-          set -x
-          max_retry=3; delay=30; i=1
-          while true; do
-            for pom in pom.xml scala2.13/pom.xml
-            do
-              mvn ${{ env.COMMON_MVN_FLAGS }} --file $pom help:evaluate -pl dist \
-                -Dexpression=included_buildvers \
-                -DforceStdout -PnoSnapshots -q | tr -d ',' | \
-                  xargs -n 1 bash -c \
-                    'mvn ${{ env.COMMON_MVN_FLAGS }} --file $1 -Dbuildver=$2 de.qaware.maven:go-offline-maven-plugin:resolve-dependencies' _ $pom
-
-              # compile base versions to cache scala compiler and compiler bridge
-              mvn ${{ env.COMMON_MVN_FLAGS }} --file $pom \
-                process-test-resources -pl sql-plugin-api -am
-            done && break || {
-            if [[ $i -le $max_retry ]]; then
-              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
-            else
-              echo "mvn command failed. Exit 1"; exit 1
-            fi
-          }
-          done
+          . .github/workflows/mvn-verify-check/populate-daily-cache.sh
       - name: all shim versions
-        id: allShimVersionsStep
+        id: all212ShimVersionsStep
         run: |
           set -x
           . jenkins/version-def.sh
@@ -113,30 +93,12 @@ jobs:
           jdkHeadVersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":8}" "${SPARK_BASE_SHIM_VERSION}")
           # jdk11
           jdk11VersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":11}" "${SPARK_SHIM_VERSIONS_JDK11[@]}")
-          # jdk17
-          jdk17VersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":17}" "${SPARK_SHIM_VERSIONS_JDK17[@]}")
           # jdk
-          jdkVersionArrBody=$jdkHeadVersionArrBody$jdk11VersionArrBody$jdk17VersionArrBody
+          jdkVersionArrBody=$jdkHeadVersionArrBody$jdk11VersionArrBody
           jdkVersionArrBody=${jdkVersionArrBody:1}
           jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody)
           echo "jdkVersions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
 
-          SCALA_BINARY_VER=2.13
-          . jenkins/version-def.sh
-          svArrBodyNoSnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":false}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS[@]}")
-          svArrBodyNoSnapshot=${svArrBodyNoSnapshot:1}
-          # get private artifact version
-          privateVer=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-private.version -DforceStdout)
-          # do not add empty snapshot versions or when private version is released one (does not include snapshot shims)
-          if [[ ${#SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]} -gt 0 && $privateVer == *"-SNAPSHOT" ]]; then
-            svArrBodySnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":true}" "${SPARK_SHIM_VERSIONS_SNAPSHOTS_ONLY[@]}")
-            svArrBodySnapshot=${svArrBodySnapshot:1}
-            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot,$svArrBodySnapshot)
-          else
-            svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot)
-          fi
-
-          echo "scala213Versions=$svJsonStr" >> $GITHUB_OUTPUT
 
   package-tests:
     needs: cache-dependencies
@@ -187,27 +149,82 @@ jobs:
           }
           done
 
+  cache-dependencies-scala213:
+    runs-on: ubuntu-latest
+    outputs:
+      scala213dailyCacheKey: ${{ steps.generateCacheKey.outputs.scala213dailyCacheKey }}
+      scala213Versions: ${{ steps.all213ShimVersionsStep.outputs.scala213Versions }}
+      sparkJDK17Versions: ${{ steps.all213ShimVersionsStep.outputs.jdkVersions }}
+    steps:
+      - uses: actions/checkout@v4 # refs/pull/:prNumber/merge
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+      - name: Generate daily cache key
+        id: generateCacheKey
+        run: |
+          set -x
+          cacheKey="${{ runner.os }}-maven-scala213-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')"
+          echo "scala213dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: ${{ env.scala213dailyCacheKey }}
+          restore-keys: ${{ runner.os }}-maven-
+      - name: populate-daily-cache
+        if: steps.cache.outputs.cache-hit != 'true'
+        env:
+          SCALA_VER: '2.13'
+        run: |
+          . .github/workflows/mvn-verify-check/populate-daily-cache.sh
+      - name: all 213 shim verions
+        id: all213ShimVersionsStep
+        run: |
+          set -x
+          SCALA_BINARY_VER=2.13
+          . jenkins/version-def.sh
+          svArrBodyNoSnapshot=$(printf ",{\"spark-version\":\"%s\",\"isSnapshot\":false}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS[@]}")
+          svArrBodyNoSnapshot=${svArrBodyNoSnapshot:1}
+          # get private artifact version
+          privateVer=$(mvn help:evaluate -q -pl dist -Dexpression=spark-rapids-private.version -DforceStdout)
+          svJsonStr=$(printf {\"include\":[%s]} $svArrBodyNoSnapshot)
+
+          echo "scala213Versions=$svJsonStr" >> $GITHUB_OUTPUT
+
+          # jdk17
+          jdk17VersionArrBody=$(printf ",{\"spark-version\":\"%s\",\"java-version\":17}" "${SPARK_SHIM_VERSIONS_JDK17_SCALA213[@]}")
+
+          jdkVersionArrBody=$jdk17VersionArrBody
+          jdkVersionArrBody=${jdkVersionArrBody:1}
+          jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody)
+          echo "jdkVersions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT
+
+
   package-tests-scala213:
-    needs: cache-dependencies
+    needs: cache-dependencies-scala213
     continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
-      matrix: ${{ fromJSON(needs.cache-dependencies.outputs.scala213Versions) }}
+      matrix: ${{ fromJSON(needs.cache-dependencies-scala213.outputs.scala213Versions) }}
       fail-fast: false
     runs-on: ubuntu-latest
     steps:
+
       - uses: actions/checkout@v4 # refs/pull/:prNumber/merge
 
       - name: Setup Java and Maven Env
         uses: actions/setup-java@v4
         with:
           distribution: adopt
-          java-version: 8
+          java-version: 17
 
       - name: Cache local Maven repository
         uses: actions/cache@v4
         with:
           path: ~/.m2
-          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+          key: ${{ needs.cache-dependencies-scala213.outputs.scala213dailyCacheKey }}
 
       - name: check runtime before tests
         run: |
@@ -218,7 +235,7 @@ jobs:
         run: |
           # https://github.com/NVIDIA/spark-rapids/issues/8847
           # specify expected versions
-          export JAVA_HOME=${JAVA_HOME_8_X64}
+          export JAVA_HOME=${JAVA_HOME_17_X64}
           export PATH=${JAVA_HOME}/bin:${PATH}
           java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
           # verify Scala 2.13 build files
@@ -246,8 +263,63 @@ jobs:
           }
           done
 
+  verify-213-modules:
+    needs: cache-dependencies-scala213
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.cache-dependencies-scala213.outputs.sparkJDK17Versions) }}
+    steps:
+      - uses: actions/checkout@v4 # refs/pull/:prNumber/merge
+
+      - name: Setup Java and Maven Env
+        uses: actions/setup-java@v4
+        with:
+          distribution: adopt
+          java-version: 17
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies-scala213.outputs.scala213dailyCacheKey }}
+
+      - name: check runtime before tests
+        run: |
+          env | grep JAVA
+          java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
+
+      - name: Build JDK
+        run: |
+          # https://github.com/NVIDIA/spark-rapids/issues/8847
+          # specify expected versions
+          export JAVA_HOME=${JAVA_HOME_${{ matrix.java-version }}_X64}
+          export PATH=${JAVA_HOME}/bin:${PATH}
+          java -version && mvn --version && echo "ENV JAVA_HOME: $JAVA_HOME, PATH: $PATH"
+           # verify Scala 2.13 build files
+          ./build/make-scala-version-build-files.sh 2.13
+          # verify git status
+          if [ -n "$(echo -n $(git status -s | grep 'scala2.13'))" ]; then
+              git add -N scala2.13/* && git diff 'scala2.13/*'
+              echo "Generated Scala 2.13 build files don't match what's in repository"
+              exit 1
+          fi
+          # change to Scala 2.13 Directory
+          cd scala2.13
+          # test command, will retry for 3 times if failed.
+          max_retry=3; delay=30; i=1
+          while true; do
+            mvn verify \
+              -P "individual,pre-merge,source-javadoc" -Dbuildver=${{ matrix.spark-version }} \
+              ${{ env.COMMON_MVN_FLAGS }} && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done
 
-  verify-all-modules:
+  verify-all-212-modules:
     needs: cache-dependencies
     runs-on: ubuntu-latest
     strategy: