Merge branch-24.04 into main

Author: jenkins

.github/workflows/add-to-project.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ jobs:
     name: Add new issues and pull requests to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v0.6.1
         with:
           project-url: https://github.com/orgs/NVIDIA/projects/4
           github-token: ${{ secrets.PROJECT_TOKEN }}

.github/workflows/auto-merge.yml

@@ -18,20 +18,20 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-24.02
+      - branch-24.04
     types: [closed]
 
 env:
-  HEAD: branch-24.02
-  BASE: branch-24.04
+  HEAD: branch-24.04
+  BASE: branch-24.06
 
 jobs:
   auto-merge:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ env.HEAD }} # force to fetch from latest upstream instead of PR ref
           token: ${{ secrets.AUTOMERGE_TOKEN }} # workaround auto-merge token to avoid GITHUB_TOKEN insufficient permission

.github/workflows/blossom-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -55,7 +55,7 @@ jobs:
       wbo4958,\
       wjxiz1992,\
       sperlingxx,\
-      pxLi,\
+      YanxuanLiu,\
       hyperbolic2346,\
       gerashegalov,\
       ttnghia,\
@@ -65,6 +65,7 @@ jobs:
       NVnavkumar,\
       yinqingh,\
       thirtiseven,\
+      parthosa,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/clang-format.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: "3.10"
     - uses: pre-commit/[email protected]

.github/workflows/signoff-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: sigoff-check job
         uses: ./.github/workflows/signoff-check

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-24.02
+	branch = branch-24.04

CONTRIBUTING.md

@@ -160,7 +160,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.02/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.04/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall

build-libcudf.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <!--
-  Copyright (c) 2022, NVIDIA CORPORATION.
+  Copyright (c) 2022-2024, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -40,13 +40,16 @@
       <arg value="-DBUILD_TESTS=OFF"/>
       <arg value="-DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS}"/>
       <arg value="-DCMAKE_INSTALL_PREFIX=${libcudf.install.path}"/>
+      <arg value="-DCUDF_DEPENDENCY_PIN_MODE=${libcudf.dependency.mode}"/>
       <arg value="-DCUDA_STATIC_RUNTIME=ON"/>
       <arg value="-DCUDF_ENABLE_ARROW_S3=OFF"/>
       <arg value="-DCUDF_ENABLE_ARROW_PARQUET=ON"/>
       <arg value="-DCUDF_USE_ARROW_STATIC=ON"/>
       <arg value="-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=${CUDF_USE_PER_THREAD_DEFAULT_STREAM}" />
+      <arg value="-DLIBCUDF_LOGGING_LEVEL=${RMM_LOGGING_LEVEL}" />
       <arg value="-DRMM_LOGGING_LEVEL=${RMM_LOGGING_LEVEL}" />
       <arg value="-DUSE_GDS=${USE_GDS}" />
+      <arg value="-C=${cudf.pin.path}/setup.cmake"/>
     </exec>
 
     <exec dir="${libcudf.build.path}"

build/build-info

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,18 +20,42 @@
 # Arguments:
 #   version  - The current version of the project
 #   git_path - The path to the repository
+#   libcudf_path - The path to the libcudf library
 set -e
+set -o pipefail
 
 echo_build_properties() {
   version=$1
   git_path=$2
-  shift 2
+  libcudf_path=$3
+  shift 3
   echo version=$version
-  echo user=$USER
+  echo user=$(whoami)
   echo revision=$(cd "$git_path" && git rev-parse HEAD)
   echo branch=$(cd "$git_path" && git rev-parse --abbrev-ref HEAD)
   echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ)
   echo url=$(cd "$git_path" && git config --get remote.origin.url)
+  gpu_architectures=$(
+    cuobjdump "$libcudf_path" | gawk '
+      match($0, /arch = sm_([0-9]+)/, tmp) {
+        arch[tmp[1]] = 1
+      }
+
+      END {
+        n = asorti(arch)
+        if (n == 0) {
+          print "ERROR: No fatbin ELF / PTX code sections found" > "/dev/stderr"
+          exit(1)
+        }
+        s = arch[1]
+        for (i = 2; i <= n; i++) {
+          s = s ";" arch[i]
+        }
+        print s
+      }
+    '
+  )
+  echo "gpu_architectures=$gpu_architectures"
   for arg in "$@"; do
     echo $arg
   done

ci/Jenkinsfile.premerge

@@ -1,6 +1,6 @@
 #!/usr/local/env groovy
 /*
- * Copyright (c) 2022-2023, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ pipeline {
     agent {
         kubernetes {
             label "premerge-init-${BUILD_TAG}"
-            cloud 'sc-ipp-blossom-prod'
+            cloud "${common.CLOUD_NAME}"
             yaml cpuImage
         }
     }
@@ -93,7 +93,7 @@ pipeline {
 
                     def title = githubHelper.getIssue().title
                     if (title ==~ /.*\[skip ci\].*/) {
-                        githubHelper.updateCommitStatus("$BUILD_URL", "Skipped", GitHubCommitState.SUCCESS)
+                        githubHelper.updateCommitStatus("", "Skipped", GitHubCommitState.SUCCESS)
                         currentBuild.result == "SUCCESS"
                         skipped = true
                         return
@@ -113,7 +113,7 @@ pipeline {
             agent {
                 kubernetes {
                     label "premerge-docker-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getDockerBuildYAML()
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -122,7 +122,7 @@ pipeline {
 
             steps {
                 script {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Running", GitHubCommitState.PENDING)
+                    githubHelper.updateCommitStatus("", "Running", GitHubCommitState.PENDING)
                     checkout(
                         changelog: false,
                         poll: true,
@@ -200,7 +200,7 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
             agent {
                 kubernetes {
                     label "premerge-ci-1-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getGPUYAML("${IMAGE_PREMERGE}", "${env.GPU_RESOURCE}", '8', '32Gi')
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -229,14 +229,15 @@ git --no-pager diff --name-only HEAD \$BASE -- ${PREMERGE_DOCKERFILE} || true"""
                 }
 
                 if (currentBuild.currentResult == "SUCCESS") {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Success", GitHubCommitState.SUCCESS)
+                    githubHelper.updateCommitStatus("", "Success", GitHubCommitState.SUCCESS)
                 } else {
                     // upload log only in case of build failure
                     def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com"]
                     guardWords.add("nvidia-smi(?s)(.*?)(?=git)") // hide GPU info
+                    guardWords.add("sc-ipp*") // hide cloud info
                     githubHelper.uploadLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)
 
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
+                    githubHelper.updateCommitStatus("", "Fail", GitHubCommitState.FAILURE)
                 }
 
                 if (TEMP_IMAGE_BUILD) {