Merge pull request #1947 from NVIDIA/branch-24.04-to-main

Merge branch-24.04 into main [skip ci]

.github/workflows/add-to-project.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ jobs:
     name: Add new issues and pull requests to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v0.6.1
         with:
           project-url: https://github.com/orgs/NVIDIA/projects/4
           github-token: ${{ secrets.PROJECT_TOKEN }}

.github/workflows/auto-merge.yml

@@ -18,20 +18,20 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-24.02
+      - branch-24.04
     types: [closed]
 
 env:
-  HEAD: branch-24.02
-  BASE: branch-24.04
+  HEAD: branch-24.04
+  BASE: branch-24.06
 
 jobs:
   auto-merge:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ env.HEAD }} # force to fetch from latest upstream instead of PR ref
           token: ${{ secrets.AUTOMERGE_TOKEN }} # workaround auto-merge token to avoid GITHUB_TOKEN insufficient permission
@@ -42,13 +42,17 @@ jobs:
           git config user.email "[email protected] "
           git fetch origin ${HEAD} ${BASE}
           git checkout -b ${INTERMEDIATE_HEAD} origin/${HEAD}
-          OUT=$(git --no-pager diff --name-only origin/${BASE} | grep "${FILE_USE_BASE}" || true)
-          [[ ! -z "${OUT}" ]] && git checkout origin/${BASE} -- ${FILE_USE_BASE} && \
-            git commit -s -am "Auto-merge use submodule in BASE ref"
+          # Sync the $BASE branch with the commits from the $HEAD branch,
+          # excluding the paths defined as $FILE_USE_BASE (located under ./thirdparty).
+          git checkout origin/${BASE} -- ${FILE_USE_BASE}
+          # If any submodule file is updated in the HEAD branch,
+          # always change it to the corresponding one from the BASE branch.
+          [ ! -z "$(git status --porcelain=v1 --untracked=no)" ] && \
+            git commit -s -am "Auto-merge use ${BASE} versions"
           git push origin ${INTERMEDIATE_HEAD} -f
         env:
           INTERMEDIATE_HEAD: bot-auto-merge-${{ env.HEAD }}
-          FILE_USE_BASE: thirdparty/cudf
+          FILE_USE_BASE: thirdparty/cudf thirdparty/cudf-pins
 
       - name: auto-merge job
         uses: ./.github/workflows/action-helper

.github/workflows/blossom-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -55,7 +55,7 @@ jobs:
       wbo4958,\
       wjxiz1992,\
       sperlingxx,\
-      pxLi,\
+      YanxuanLiu,\
       hyperbolic2346,\
       gerashegalov,\
       ttnghia,\
@@ -65,6 +65,7 @@ jobs:
       NVnavkumar,\
       yinqingh,\
       thirtiseven,\
+      parthosa,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/clang-format.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
     - uses: pre-commit/[email protected]

.github/workflows/signoff-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: sigoff-check job
         uses: ./.github/workflows/signoff-check

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-24.02
+	branch = branch-24.04

CONTRIBUTING.md

@@ -160,7 +160,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.02/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.04/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall

build-libcudf.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <!--
-  Copyright (c) 2022, NVIDIA CORPORATION.
+  Copyright (c) 2022-2024, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -40,13 +40,16 @@
       <arg value="-DBUILD_TESTS=OFF"/>
       <arg value="-DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS}"/>
       <arg value="-DCMAKE_INSTALL_PREFIX=${libcudf.install.path}"/>
+      <arg value="-DCUDF_DEPENDENCY_PIN_MODE=${libcudf.dependency.mode}"/>
       <arg value="-DCUDA_STATIC_RUNTIME=ON"/>
       <arg value="-DCUDF_ENABLE_ARROW_S3=OFF"/>
       <arg value="-DCUDF_ENABLE_ARROW_PARQUET=ON"/>
       <arg value="-DCUDF_USE_ARROW_STATIC=ON"/>
       <arg value="-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=${CUDF_USE_PER_THREAD_DEFAULT_STREAM}" />
+      <arg value="-DLIBCUDF_LOGGING_LEVEL=${RMM_LOGGING_LEVEL}" />
       <arg value="-DRMM_LOGGING_LEVEL=${RMM_LOGGING_LEVEL}" />
       <arg value="-DUSE_GDS=${USE_GDS}" />
+      <arg value="-C=${cudf.pin.path}/setup.cmake"/>
     </exec>
 
     <exec dir="${libcudf.build.path}"

build/build-info

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,18 +20,42 @@
 # Arguments:
 #   version  - The current version of the project
 #   git_path - The path to the repository
+#   libcudf_path - The path to the libcudf library
 set -e
+set -o pipefail
 
 echo_build_properties() {
   version=$1
   git_path=$2
-  shift 2
+  libcudf_path=$3
+  shift 3
   echo version=$version
-  echo user=$USER
+  echo user=$(whoami)
   echo revision=$(cd "$git_path" && git rev-parse HEAD)
   echo branch=$(cd "$git_path" && git rev-parse --abbrev-ref HEAD)
   echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ)
   echo url=$(cd "$git_path" && git config --get remote.origin.url)
+  gpu_architectures=$(
+    cuobjdump "$libcudf_path" | gawk '
+      match($0, /arch = sm_([0-9]+)/, tmp) {
+        arch[tmp[1]] = 1
+      }
+
+      END {
+        n = asorti(arch)
+        if (n == 0) {
+          print "ERROR: No fatbin ELF / PTX code sections found" > "/dev/stderr"
+          exit(1)
+        }
+        s = arch[1]
+        for (i = 2; i <= n; i++) {
+          s = s ";" arch[i]
+        }
+        print s
+      }
+    '
+  )
+  echo "gpu_architectures=$gpu_architectures"
   for arg in "$@"; do
     echo $arg
   done

ci/Jenkinsfile.premerge

@@ -1,6 +1,6 @@
 #!/usr/local/env groovy
 /*
- * Copyright (c) 2022-2023, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ pipeline {
     agent {
         kubernetes {
             label "premerge-init-${BUILD_TAG}"
-            cloud 'sc-ipp-blossom-prod'
+            cloud "${common.CLOUD_NAME}"
             yaml cpuImage
         }
     }
@@ -93,7 +93,7 @@ pipeline {
 
                     def title = githubHelper.getIssue().title
                     if (title ==~ /.*\[skip ci\].*/) {
-                        githubHelper.updateCommitStatus("$BUILD_URL", "Skipped", GitHubCommitState.SUCCESS)
+                        githubHelper.updateCommitStatus("", "Skipped", GitHubCommitState.SUCCESS)
                         currentBuild.result == "SUCCESS"
                         skipped = true
                         return
@@ -113,7 +113,7 @@ pipeline {
             agent {
                 kubernetes {
                     label "premerge-docker-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getDockerBuildYAML()
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -122,7 +122,7 @@ pipeline {
 
             steps {
                 script {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Running", GitHubCommitState.PENDING)
+                    githubHelper.updateCommitStatus("", "Running", GitHubCommitState.PENDING)
                     checkout(
                         changelog: false,
                         poll: true,
@@ -200,7 +200,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
             agent {
                 kubernetes {
                     label "premerge-ci-1-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getGPUYAML("${IMAGE_PREMERGE}", "${env.GPU_RESOURCE}", '8', '32Gi')
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -229,14 +229,15 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
                 }
 
                 if (currentBuild.currentResult == "SUCCESS") {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Success", GitHubCommitState.SUCCESS)
+                    githubHelper.updateCommitStatus("", "Success", GitHubCommitState.SUCCESS)
                 } else {
                     // upload log only in case of build failure
                     def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com"]
                     guardWords.add("nvidia-smi(?s)(.*?)(?=git)") // hide GPU info
+                    guardWords.add("sc-ipp*") // hide cloud info
                     githubHelper.uploadLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)
 
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
+                    githubHelper.updateCommitStatus("", "Fail", GitHubCommitState.FAILURE)
                 }
 
                 if (TEMP_IMAGE_BUILD) {

ci/deploy.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -69,14 +69,15 @@ IFS="$ORI_IFS"
 FIRST_FILE=${CLASS_FILES%%,*}
 cp -f "$FIRST_FILE" "$FPATH.jar"
 
+GPG_PLUGIN="org.apache.maven.plugins:maven-gpg-plugin:3.1.0:sign-and-deploy-file"
 ###### Build the deploy command ######
 if [ "$SIGN_FILE" == true ]; then
     case $SIGN_TOOL in
         nvsec)
-            DEPLOY_CMD="$MVN gpg:sign-and-deploy-file -Dgpg.executable=nvsec_sign"
+            DEPLOY_CMD="$MVN $GPG_PLUGIN -Dgpg.executable=nvsec_sign"
             ;;
         gpg)
-            DEPLOY_CMD="$MVN gpg:sign-and-deploy-file -Dgpg.passphrase=$GPG_PASSPHRASE "
+            DEPLOY_CMD="$MVN $GPG_PLUGIN -Dgpg.passphrase=$GPG_PASSPHRASE "
             ;;
         *)
             echo "Error unsupported sign type : $SIGN_TYPE !"

ci/submodule-sync.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,8 +49,14 @@ else
   git merge origin/${REF}
 fi
 
-# sync up cudf from remote
-git submodule update --remote --merge
+# sync up cudf from remote, checkout to cudf release tag if CUDF_TAG is set
+if [ -n "$CUDF_TAG" ]; then
+  pushd thirdparty/cudf
+  git checkout tags/$CUDF_TAG
+  popd
+else
+  git submodule update --remote --merge
+fi
 cudf_sha=$(git -C thirdparty/cudf rev-parse HEAD)
 if [[ "${cudf_sha}" == "${cudf_prev_sha}" ]]; then
   echo "Submodule is up to date."
@@ -59,8 +65,6 @@ fi
 
 echo "Try update cudf submodule to ${cudf_sha}..."
 git add .
-git diff-index --quiet HEAD || git commit -s -m "Update submodule cudf to ${cudf_sha}"
-sha=$(git rev-parse HEAD)
 
 echo "Test against ${cudf_sha}..."
 
@@ -69,6 +73,7 @@ set +e
 ${MVN} verify ${MVN_MIRROR} \
   -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
   -Dlibcudf.build.configure=true \
+  -Dlibcudf.dependency.mode=latest \
   -DUSE_GDS=ON -Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest \
   -DBUILD_TESTS=ON \
   -DUSE_SANITIZER=ON
@@ -83,6 +88,18 @@ else
   echo "Test failed, will update the result"
 fi
 
+# Extract the rapids-cmake sha1 that we need to pin too
+rapids_cmake_sha=$(git -C thirdparty/cudf/cpp/build/_deps/rapids-cmake-src/ rev-parse HEAD)
+echo "Update rapids-cmake pinned SHA1 to ${rapids_cmake_sha}"
+echo "${rapids_cmake_sha}" > thirdparty/cudf-pins/rapids-cmake.sha
+
+# Do the git add after the build so that we get
+# the updated versions.json generated by the build
+echo "Update cudf submodule to ${cudf_sha} with updated pinned versions"
+git add .
+git diff-index --quiet HEAD || git commit -s -m "Update submodule cudf to ${cudf_sha}"
+sha=$(git rev-parse HEAD)
+
 # push the intermediate branch and create PR against REF
 # if test passed, it will try auto-merge the PR
 # if test failed, it will only comment the test result in the PR