Merge pull request #2480 from NVIDIA/merge-branch-24.10-to-main

Merge branch-24.10 into main

.github/workflows/auto-merge.yml

@@ -18,25 +18,57 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-24.08
+      - branch-*
     types: [closed]
 
-env:
-  HEAD: branch-24.08
-  BASE: branch-24.10
-
 jobs:
-  auto-merge:
+  auto-merge: # TODO: use spark-rapids-common shared action when available
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
 
     steps:
+      - name: set HEAD ENV
+        run: echo "HEAD=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
+
+      - name: Generate target branch
+        run: |
+          current_branch="${{ env.HEAD }}"
+          version=${current_branch#branch-}
+
+          IFS='.' read -r -a parts <<< "$version"
+          year=${parts[0]}
+          month=${parts[1]}
+          month=$((10#$month + 2))
+          if [ $month -gt 12 ]; then
+            month=$((month - 12))
+            year=$((year + 1))
+          fi
+
+          next_release=$(printf "%02d.%02d" $year $month)
+          echo "Next release is $next_release"
+          echo "BASE=branch-$next_release" >> $GITHUB_ENV
+
+      - name: Check if target branch exists
+        run: |
+          CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.AUTOMERGE_TOKEN }}" \
+            https://api.github.com/repos/${{ github.repository }}/branches/${{ env.BASE }})
+          echo "Response code: $CODE..."
+
+          if [ $CODE -eq 200 ]; then
+            echo "branch_exists=true" >> $GITHUB_ENV
+          else
+            echo "branch_exists=false" >> $GITHUB_ENV
+            echo "Failed to find ${{ env.BASE }}. Skip auto-merge..."
+          fi
+
       - uses: actions/checkout@v4
+        if: env.branch_exists == 'true'
         with:
           ref: ${{ env.HEAD }} # force to fetch from latest upstream instead of PR ref
           token: ${{ secrets.AUTOMERGE_TOKEN }} # workaround auto-merge token to avoid GITHUB_TOKEN insufficient permission
 
       - name: push intermediate branch for auto-merge
+        if: env.branch_exists == 'true'
         run: |
           git config user.name "spark-rapids automation"
           git config user.email "[email protected] "
@@ -55,6 +87,7 @@ jobs:
           FILE_USE_BASE: thirdparty/cudf thirdparty/cudf-pins
 
       - name: auto-merge job
+        if: env.branch_exists == 'true'
         uses: ./.github/workflows/action-helper
         with:
           operator: auto-merge
@@ -64,4 +97,3 @@ jobs:
           HEAD: bot-auto-merge-${{ env.HEAD }}
           BASE: ${{ env.BASE }}
           TOKEN: ${{ secrets.AUTOMERGE_TOKEN }}
-

.github/workflows/blossom-ci.yml

@@ -68,7 +68,9 @@ jobs:
         github.actor == 'binmahone' ||
         github.actor == 'pmattione-nvidia' ||
         github.actor == 'Feng-Jiang28' ||
-        github.actor == 'pxLi'
+        github.actor == 'pxLi' ||
+        github.actor == 'ustcfy' ||
+        github.actor == 'zpuller'
       )
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/signoff-check.yml

@@ -23,12 +23,10 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - name: sigoff-check job
-        uses: ./.github/workflows/signoff-check
-        env:
-          OWNER: NVIDIA
-          REPO_NAME: spark-rapids-jni
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PULL_NUMBER: ${{ github.event.number }}
+      - name: signoff
+        uses: NVIDIA/spark-rapids-common/signoff-check@main
+        with:
+          owner: ${{ github.repository_owner }}
+          repo: spark-rapids-jni
+          pull_number: ${{ github.event.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-24.08
+	branch = branch-24.10

CONTRIBUTING.md

@@ -165,7 +165,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.08/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.10/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall
@@ -299,7 +299,7 @@ then run `cuda-gdb`. You do not necessarily need to run `cuda-gdb` in Docker):
 
 ```bash
 ./build/run-in-docker
-bash-4.2$ cuda-gdb target/cmake-build/gtests/ROW_CONVERSION
+bash-4.2$ cuda-gdb target/jni/cmake-build/gtests/ROW_CONVERSION
 ```
 
 You can also use the [NVIDIA Nsight VSCode Code Integration](https://docs.nvidia.com/nsight-visual-studio-code-edition/cuda-debugger/index.html)

build/apply-patches

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Run a command in a Docker container with devtoolset
+
+set -e
+
+BASE_DIR=$( git rev-parse --show-toplevel )
+
+PATCH_DIR=${PATCH_DIR:-$(realpath "$BASE_DIR/patches/")}
+
+CUDF_DIR=${CUDF_DIR:-$(realpath "$BASE_DIR/thirdparty/cudf/")}
+
+pushd "$CUDF_DIR"
+if [ -n "$(git status --porcelain --untracked-files=no)" ] ; then
+    echo "Error: CUDF repository has uncommitted changes. No patches will be applied..."
+    exit 1
+fi
+
+find "$PATCH_DIR" -maxdepth 1 -type f -print0 | sort -zV | while IFS= read -r -d '' file; do
+    echo "patching with: $file"
+    patch --no-backup-if-mismatch -f -t --reject-file=- -p1 -i "$file"
+done
+popd

build/unapply-patches

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Run a command in a Docker container with devtoolset
+
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+PATCH_DIR=${PATCH_DIR:-$(realpath "$SCRIPT_DIR/../patches/")}
+
+CUDF_DIR=${CUDF_DIR:-$(realpath "$SCRIPT_DIR/../thirdparty/cudf/")}
+
+
+pushd "$CUDF_DIR"
+if [ -n "$(git status --porcelain --untracked-files=no)" ] ; then
+  #only try to remove patches if it looks like something was changed
+  find "$PATCH_DIR" -maxdepth 1 -type f -print0 | sort -zV -r | while IFS= read -r -d '' file; do
+      echo "patching with: $file"
+      patch -R --no-backup-if-mismatch --reject-file=- -f -t -p1 -i "$file"
+  done
+fi
+
+# Check for modifications
+if [ -n "$(git status --porcelain --untracked-files=no)" ] ; then
+    echo "Error: CUDF repository has uncommitted changes. You might want to clean in manually if you know that is expected"
+    exit 1
+fi
+popd