Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: rapidsai/dask-cuda
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v22.10.00
Choose a base ref
...
head repository: rapidsai/dask-cuda
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
Loading
Showing with 5,731 additions and 4,797 deletions.
  1. +0 −5 .coveragerc
  2. +14 −0 .flake8
  3. +0 −1 .gitattributes
  4. +11 −6 .github/CODEOWNERS
  5. +5 −0 .github/copy-pr-bot.yaml
  6. +1 −1 .github/labeler.yml
  7. +2 −1 .github/ops-bot.yaml
  8. +81 −0 .github/workflows/build.yaml
  9. +1 −1 .github/workflows/labeler.yml
  10. +63 −0 .github/workflows/pr.yaml
  11. +0 −57 .github/workflows/stale.yaml
  12. +33 −0 .github/workflows/test.yaml
  13. +26 −0 .github/workflows/trigger-breaking-change-alert.yaml
  14. +41 −3 .pre-commit-config.yaml
  15. +0 −7 .readthedocs.yml
  16. +431 −0 CHANGELOG.md
  17. +1 −2 MANIFEST.in
  18. +2 −4 README.md
  19. +1 −0 VERSION
  20. +38 −0 ci/build_docs.sh
  21. +20 −0 ci/build_python.sh
  22. +13 −0 ci/build_wheel.sh
  23. +18 −0 ci/check_style.sh
  24. +0 −61 ci/checks/style.sh
  25. +0 −96 ci/cpu/build.sh
  26. +0 −16 ci/cpu/upload-pypi.sh
  27. +0 −42 ci/cpu/upload.sh
  28. +0 −50 ci/docs/build.sh
  29. +0 −125 ci/gpu/build.sh
  30. +0 −58 ci/local/README.md
  31. +0 −145 ci/local/build.sh
  32. +46 −10 ci/release/update-version.sh
  33. +66 −0 ci/run_benchmarks.sh
  34. +20 −0 ci/run_pytest.sh
  35. +60 −0 ci/test_python.sh
  36. +24 −0 ci/test_wheel.sh
  37. +18 −0 ci/validate_wheel.sh
  38. +0 −1 codecov.yml
  39. +36 −0 conda/environments/all_cuda-114_arch-x86_64.yaml
  40. +36 −0 conda/environments/all_cuda-118_arch-x86_64.yaml
  41. +37 −0 conda/environments/all_cuda-128_arch-x86_64.yaml
  42. +0 −9 conda/environments/builddocs_py37.yml
  43. +0 −3 conda/recipes/dask-cuda/build.sh
  44. +34 −13 conda/recipes/dask-cuda/meta.yaml
  45. +1 −0 dask_cuda/VERSION
  46. +24 −14 dask_cuda/__init__.py
  47. +28 −554 dask_cuda/_version.py
  48. +36 −15 dask_cuda/benchmarks/common.py
  49. +23 −19 dask_cuda/benchmarks/local_cudf_groupby.py
  50. +42 −40 dask_cuda/benchmarks/local_cudf_merge.py
  51. +114 −43 dask_cuda/benchmarks/local_cudf_shuffle.py
  52. +21 −19 dask_cuda/benchmarks/local_cupy.py
  53. +21 −19 dask_cuda/benchmarks/local_cupy_map_overlap.py
  54. +267 −0 dask_cuda/benchmarks/read_parquet.py
  55. +243 −22 dask_cuda/benchmarks/utils.py
  56. +182 −52 dask_cuda/{cli/dask_cuda_worker.py → cli.py}
  57. 0 dask_cuda/cli/__init__.py
  58. +40 −29 dask_cuda/cuda_worker.py
  59. +39 −19 dask_cuda/device_host_file.py
  60. +33 −29 dask_cuda/disk_io.py
  61. +69 −4 dask_cuda/explicit_comms/comms.py
  62. +520 −267 dask_cuda/explicit_comms/dataframe/shuffle.py
  63. +7 −2 dask_cuda/get_device_memory_objects.py
  64. +52 −18 dask_cuda/initialize.py
  65. +55 −0 dask_cuda/is_spillable_object.py
  66. +114 −46 dask_cuda/local_cuda_cluster.py
  67. +213 −0 dask_cuda/plugins.py
  68. +6 −4 dask_cuda/proxify_device_objects.py
  69. +65 −31 dask_cuda/proxify_host_file.py
  70. +31 −31 dask_cuda/proxy_object.py
  71. +4 −0 dask_cuda/tests/pytest.ini
  72. +153 −0 dask_cuda/tests/test_cudf_builtin_spilling.py
  73. +306 −34 dask_cuda/tests/test_dask_cuda_worker.py
  74. +13 −18 dask_cuda/tests/test_device_host_file.py
  75. +58 −18 dask_cuda/tests/test_dgx.py
  76. +160 −86 dask_cuda/tests/test_explicit_comms.py
  77. +22 −0 dask_cuda/tests/test_from_array.py
  78. +2 −2 dask_cuda/tests/test_gds.py
  79. +69 −21 dask_cuda/tests/test_initialize.py
  80. +244 −28 dask_cuda/tests/test_local_cuda_cluster.py
  81. +93 −59 dask_cuda/tests/test_proxify_host_file.py
  82. +59 −39 dask_cuda/tests/test_proxy.py
  83. +209 −46 dask_cuda/tests/test_spill.py
  84. +29 −10 dask_cuda/tests/test_utils.py
  85. +12 −0 dask_cuda/tests/test_version.py
  86. +242 −116 dask_cuda/utils.py
  87. +45 −0 dask_cuda/utils_test.py
  88. +2 −1 dask_cuda/worker_spec.py
  89. +207 −0 dependencies.yaml
  90. +1 −1 docs/Makefile
  91. +0 −4 docs/requirements.txt
  92. +14 −3 docs/source/api.rst
  93. +3 −2 docs/source/conf.py
  94. +10 −4 docs/source/examples/best-practices.rst
  95. +18 −18 docs/source/examples/ucx.rst
  96. +4 −4 docs/source/examples/worker_count.rst
  97. +2 −2 docs/source/explicit_comms.rst
  98. +2 −2 docs/source/index.rst
  99. +2 −2 docs/source/install.rst
  100. +10 −6 docs/source/quickstart.rst
  101. +90 −11 docs/source/spilling.rst
  102. +57 −0 docs/source/troubleshooting.rst
  103. +14 −5 docs/source/ucx.rst
  104. +3 −3 examples/ucx/dask_cuda_worker.sh
  105. +147 −0 pyproject.toml
  106. +0 −9 pytest.ini
  107. +0 −5 requirements.txt
  108. +0 −19 rtd/Makefile
  109. +0 −211 rtd/conf.py
  110. +0 −1 rtd/index.rst
  111. +0 −11 rtd/templates/redirect.html
  112. +0 −55 setup.cfg
  113. +0 −62 setup.py
  114. +0 −1,885 versioneer.py
5 changes: 0 additions & 5 deletions .coveragerc

This file was deleted.

14 changes: 14 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[flake8]
exclude = docs, __init__.py
max-line-length = 88
ignore =
# Assigning lambda expression
E731
# Ambiguous variable names
E741
# line break before binary operator
W503
# whitespace before :
E203
# whitespace after ,
E231
1 change: 0 additions & 1 deletion .gitattributes

This file was deleted.

17 changes: 11 additions & 6 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
#python code owners
dask_cuda/ @rapidsai/daskcuda-python-codeowners

#build/ops code owners
.github/ @rapidsai/ops-codeowners
ci/ @rapidsai/ops-codeowners
conda/ @rapidsai/ops-codeowners
**/Dockerfile @rapidsai/ops-codeowners
**/.dockerignore @rapidsai/ops-codeowners
#CI code owners
/.github/ @rapidsai/ci-codeowners
/ci/ @rapidsai/ci-codeowners
/.pre-commit-config.yaml @rapidsai/ci-codeowners

#packaging code owners
/.devcontainer/ @rapidsai/packaging-codeowners
/conda/ @rapidsai/packaging-codeowners
/dependencies.yaml @rapidsai/packaging-codeowners
/build.sh @rapidsai/packaging-codeowners
pyproject.toml @rapidsai/packaging-codeowners
5 changes: 5 additions & 0 deletions .github/copy-pr-bot.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Configuration file for `copy-pr-bot` GitHub App
# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/

enabled: true
auto_sync_draft: false
2 changes: 1 addition & 1 deletion .github/labeler.yml
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
python:
- 'dask_cuda/**'

gpuCI:
ci:
- 'ci/**'

conda:
3 changes: 2 additions & 1 deletion .github/ops-bot.yaml
Original file line number Diff line number Diff line change
@@ -5,4 +5,5 @@ auto_merger: true
branch_checker: true
label_checker: true
release_drafter: true
external_contributors: false
recently_updated: true
forward_merger: true
81 changes: 81 additions & 0 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
name: build

on:
push:
branches:
- "branch-*"
tags:
- v[0-9][0-9].[0-9][0-9].[0-9][0-9]
workflow_dispatch:
inputs:
branch:
required: true
type: string
date:
required: true
type: string
sha:
required: true
type: string
build_type:
type: string
default: nightly

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true

jobs:
conda-python-build:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
docs-build:
if: github.ref_type == 'branch'
needs: [conda-python-build]
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02
with:
arch: "amd64"
branch: ${{ inputs.branch }}
build_type: ${{ inputs.build_type || 'branch' }}
container_image: "rapidsai/ci-conda:latest"
date: ${{ inputs.date }}
node_type: "gpu-v100-latest-1"
run_script: "ci/build_docs.sh"
sha: ${{ inputs.sha }}
upload-conda:
needs: [conda-python-build]
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-25.02
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
wheel-build:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
script: ci/build_wheel.sh
# Package is pure Python and only ever requires one build.
# This selects "ARCH=amd64 + the latest supported Python + CUDA".
matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
wheel-publish:
needs: wheel-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
sha: ${{ inputs.sha }}
date: ${{ inputs.date }}
package-name: dask-cuda
2 changes: 1 addition & 1 deletion .github/workflows/labeler.yml
Original file line number Diff line number Diff line change
@@ -6,6 +6,6 @@ jobs:
triage:
runs-on: ubuntu-latest
steps:
- uses: actions/labeler@main
- uses: actions/labeler@v4
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
63 changes: 63 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: pr

on:
push:
branches:
- "pull-request/[0-9]+"

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
pr-builder:
needs:
- checks
- conda-python-build
- conda-python-tests
- docs-build
- wheel-build
- wheel-tests
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02
checks:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02
conda-python-build:
needs: checks
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-25.02
with:
build_type: pull-request
conda-python-tests:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02
with:
build_type: pull-request
docs-build:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02
with:
build_type: pull-request
node_type: "gpu-v100-latest-1"
arch: "amd64"
container_image: "rapidsai/ci-conda:latest"
run_script: "ci/build_docs.sh"
wheel-build:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02
with:
build_type: pull-request
# Package is pure Python and only ever requires one build.
# This selects "ARCH=amd64 + the latest supported Python + CUDA".
matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
script: "ci/build_wheel.sh"
wheel-tests:
needs: wheel-build
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02
with:
build_type: pull-request
script: "ci/test_wheel.sh"
57 changes: 0 additions & 57 deletions .github/workflows/stale.yaml

This file was deleted.

33 changes: 33 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: test

on:
workflow_dispatch:
inputs:
branch:
required: true
type: string
date:
required: true
type: string
sha:
required: true
type: string

jobs:
conda-python-tests:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.02
with:
build_type: nightly
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
wheel-tests:
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02
with:
build_type: nightly
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
script: ci/test_wheel.sh
26 changes: 26 additions & 0 deletions .github/workflows/trigger-breaking-change-alert.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Trigger Breaking Change Notifications

on:
pull_request_target:
types:
- closed
- reopened
- labeled
- unlabeled

jobs:
trigger-notifier:
if: contains(github.event.pull_request.labels.*.name, 'breaking')
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@branch-25.02
with:
sender_login: ${{ github.event.sender.login }}
sender_avatar: ${{ github.event.sender.avatar_url }}
repo: ${{ github.repository }}
pr_number: ${{ github.event.pull_request.number }}
pr_title: "${{ github.event.pull_request.title }}"
pr_body: "${{ github.event.pull_request.body || '_Empty PR description_' }}"
pr_base_ref: ${{ github.event.pull_request.base.ref }}
pr_author: ${{ github.event.pull_request.user.login }}
event_action: ${{ github.event.action }}
pr_merged: ${{ github.event.pull_request.merged }}
44 changes: 41 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,15 +1,53 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.3.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- repo: https://github.com/pycqa/isort
rev: 5.6.4
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/ambv/black
rev: 22.3.0
hooks:
- id: black
- repo: https://gitlab.com/pycqa/flake8
rev: 3.8.3
- repo: https://github.com/PyCQA/flake8
rev: 7.1.1
hooks:
- id: flake8
- repo: https://github.com/codespell-project/codespell
rev: v2.1.0
hooks:
- id: codespell
exclude: |
(?x)^(
.*test.*|
^CHANGELOG.md$|
)
- repo: https://github.com/pre-commit/mirrors-mypy
rev: 'v0.991'
hooks:
- id: mypy
additional_dependencies: [types-cachetools]
args: ["--module=dask_cuda", "--ignore-missing-imports"]
pass_filenames: false
- repo: https://github.com/rapidsai/pre-commit-hooks
rev: v0.4.0
hooks:
- id: verify-alpha-spec
args: ["--fix", "--mode=release"]
- repo: https://github.com/rapidsai/dependency-file-generator
rev: v1.16.0
hooks:
- id: rapids-dependency-file-generator
args: ["--clean"]
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.10.0.1
hooks:
- id: shellcheck
args: ["--severity=warning"]
files: ^ci/

default_language_version:
python: python3
Loading