Skip to content

Commit

Permalink
Merge branch 'main' into link_doc_references
Browse files Browse the repository at this point in the history
  • Loading branch information
ksimpson-work authored Dec 10, 2024
2 parents d338f50 + c1fea41 commit 8303d38
Show file tree
Hide file tree
Showing 39 changed files with 2,496 additions and 1,720 deletions.
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,5 @@ cuda/_version.py export-subst
# we do not own any headers checked in, don't touch them
*.h binary
*.hpp binary
# git should not convert line endings in PNG files
*.png binary
108 changes: 57 additions & 51 deletions .github/actions/build/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,73 +13,79 @@ inputs:
host-platform:
required: true
type: string
use-container:
required: true
type: boolean
docker-image:
type: string
required: true
upload-enabled:
required: true
type: boolean
python-version:
required: true
type: string

runs:
using: composite
steps:
- name: Build cuda.core wheel
uses: pypa/[email protected]
env:
CIBW_BUILD: ${{ env.CIBW_BUILD }}
CIBW_ARCHS_LINUX: "native"
CIBW_BUILD_VERBOSITY: 1
# # ensure Python.h & co can be found
# CIBW_BEFORE_BUILD_WINDOWS: >
# python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
with:
package-dir: ./cuda_core/
output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

- if: ${{ inputs.use-container }}
name: Build (in container)
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
docker run \
-e AWS_REGION \
-e AWS_SESSION_TOKEN \
-e AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY \
-e GITHUB_TOKEN \
-e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \
-e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \
-e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
-e USE_CUDA="$USE_CUDA" \
-e REPO_DIR="$REPO_DIR" \
-e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
-e PYTHON_VERSION="$PYTHON_VERSION" \
-v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
-v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \
-v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \
--rm "${{ inputs.docker-image }}" \
/bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
- if: ${{ !inputs.use-container }}
name: Build (without container)
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
"${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
- name: Display structure of the bindings artifacts folder (post build)
- name: List the cuda.core artifacts directory
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }}
ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }}
if [[ "${{ inputs.host-platform }}" == win* ]]; then
export CHOWN=chown
else
export CHOWN="sudo chown"
fi
$CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
- name: Upload bindings build artifacts
- name: Upload cuda.core build artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ env.BINDINGS_ARTIFACT_NAME }}
path: ${{ env.BINDINGS_ARTIFACTS_DIR }}
name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
if-no-files-found: error
overwrite: 'true'

- name: Build cuda.bindings wheel
uses: pypa/[email protected]
env:
CIBW_BUILD: ${{ env.CIBW_BUILD }}
CIBW_ARCHS_LINUX: "native"
CIBW_BUILD_VERBOSITY: 1
CIBW_ENVIRONMENT_LINUX: >
CUDA_PATH="$(realpath ./cuda_toolkit)"
PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
CIBW_ENVIRONMENT_WINDOWS: >
CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))"
# PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
# # ensure Python.h & co can be found
# CIBW_BEFORE_BUILD_WINDOWS: >
# python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
with:
package-dir: ./cuda_bindings/
output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

- name: Display structure of the core artifacts folder (post build)
- name: List the cuda.bindings artifacts directory
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }}
ls -lahR ${{ env.CORE_ARTIFACTS_DIR }}
if [[ "${{ inputs.host-platform }}" == win* ]]; then
export CHOWN=chown
else
export CHOWN="sudo chown"
fi
$CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
- name: Upload core build artifacts
- name: Upload cuda.bindings build artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ env.CORE_ARTIFACT_NAME }}
path: ${{ env.CORE_ARTIFACTS_DIR }}
name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
if-no-files-found: error
overwrite: 'true'
155 changes: 134 additions & 21 deletions .github/actions/setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,51 +22,164 @@ inputs:
python-version:
required: true
type: string
cuda-version:
required: true
type: string

runs:
using: composite
steps:
- name: Set REPO_DIR and Dump environment
# WAR: setup-python is not relocatable...
# see https://github.com/actions/setup-python/issues/871
- name: Set up Python ${{ inputs.python-version }}
if: ${{ startsWith(inputs.host-platform, 'linux') }}
id: setup-python
uses: actions/setup-python@v5
with:
python-version: "3.12"

- name: Set up MSVC
if: ${{ startsWith(inputs.host-platform, 'win') }}
uses: ilammy/msvc-dev-cmd@v1

- name: Dump environment
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV
env
- name: Set environment variables
- name: Set up CTK cache variable
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
echo "CTK_CACHE_KEY=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}" >> $GITHUB_ENV
echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}.tar.gz" >> $GITHUB_ENV
- name: Download CTK cache
id: ctk-get-cache
uses: actions/cache/restore@v4
continue-on-error: true
with:
key: ${{ env.CTK_CACHE_KEY }}
path: ./${{ env.CTK_CACHE_FILENAME }}

- name: Get CUDA components
if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }}
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
CUDA_PATH="./cuda_toolkit"
mkdir $CUDA_PATH
# The binary archives (redist) are guaranteed to be updated as part of the release posting.
CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
if [[ "${{ inputs.host-platform }}" == linux* ]]; then
if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
CTK_SUBDIR="linux-x86_64"
elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
CTK_SUBDIR="linux-sbsa"
fi
function extract() {
tar -xvf $1 -C $CUDA_PATH --strip-components=1
}
elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
CTK_SUBDIR="windows-x86_64"
function extract() {
_TEMP_DIR_=$(mktemp -d)
unzip $1 -d $_TEMP_DIR_
cp -r $_TEMP_DIR_/*/* $CUDA_PATH
rm -rf $_TEMP_DIR_
}
fi
function populate_cuda_path() {
# take the component name as a argument
function download() {
curl -kLSs $1 -o $2
}
CTK_COMPONENT=$1
CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL |
python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")"
CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
extract $CTK_COMPONENT_COMPONENT_FILENAME
rm $CTK_COMPONENT_COMPONENT_FILENAME
}
# Get headers and shared libraries in place
# Note: the existing artifact would need to be manually deleted (ex: through web UI)
# if this list is changed, as the artifact actions do not offer any option for us to
# invalidate the artifact.
populate_cuda_path cuda_nvcc
populate_cuda_path cuda_cudart
populate_cuda_path cuda_nvrtc
populate_cuda_path cuda_profiler_api
populate_cuda_path libnvjitlink
ls -l $CUDA_PATH
# Prepare the cache
# Note: try to escape | and > ...
tar -czvf ${CTK_CACHE_FILENAME} ${CUDA_PATH}
# Note: the headers will be copied into the cibuildwheel manylinux container,
# so setting the CUDA_PATH env var here is meaningless.
WITH_TESTS_STR=''
if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then
WITH_TESTS_STR='-with_tests'
- name: Upload CTK cache
if: ${{ always() &&
steps.ctk-get-cache.outputs.cache-hit != 'true' }}
uses: actions/cache/save@v4
with:
key: ${{ env.CTK_CACHE_KEY }}
path: ./${{ env.CTK_CACHE_FILENAME }}

- name: Restore CTK cache
if: ${{ steps.ctk-get-cache.outputs.cache-hit == 'true' }}
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
ls -l
CUDA_PATH="./cuda_toolkit"
tar -xzvf $CTK_CACHE_FILENAME
ls -l $CUDA_PATH
if [ ! -d "$CUDA_PATH/include" ]; then
exit 1
fi
TARGET_PLATFORM='linux-64'
if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
- name: Set environment variables
shell: bash --noprofile --norc -xeuo pipefail {0}
run: |
# TODO: just align host-platform names with TARGET_PLATFORM...
if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
TARGET_PLATFORM='linux-64'
elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
TARGET_PLATFORM='linux-aarch64'
elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
TARGET_PLATFORM='win-64'
fi
BUILD_MODE="${{ inputs.build-mode }}"
BUILD_MODE_STR=""
[ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}"
PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
if [[ "${{ inputs.host-platform }}" == linux* ]]; then
CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
REPO_DIR=$(pwd)
elif [[ "${{ inputs.host-platform }}" == win* ]]; then
CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
PWD=$(pwd)
REPO_DIR=$(cygpath -w $PWD)
fi
BUILD_MODE="${{ inputs.build-mode }}"
if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
# We upload release versions in the default folder.
PKG_DIR="${TARGET_PLATFORM}"
else
PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
fi
PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV
echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV
echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV
echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
Loading

0 comments on commit 8303d38

Please sign in to comment.