Add pyproject.toml with legacy build backend to keep most logic in setup.py

.github/workflows/no-torch.yml

@@ -7,6 +7,8 @@ on:
       - 'accelerator/**'
       - '.github/workflows/no-torch.yml'
       - 'op_builder/**'
+      - 'setup.py'
+      - 'pyproject.toml'
   schedule:
     - cron: "0 0 * * *"
 

.github/workflows/nv-mii.yml

@@ -13,6 +13,7 @@ on:
       - '.github/workflows/nv-mii.yml'
       - 'requirements/**'
       - 'setup.py'
+      - 'pyproject.toml'
       - 'deepspeed/__init__.py'
       - 'deepspeed/inference/**'
       - '!deepspeed/inference/v2/**' # exclude v2 dir

.github/workflows/nv-pre-compile-ops.yml

@@ -36,7 +36,7 @@ jobs:
             #python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
         - name: Compile DeepSpeed Ops
           run: |
-            DS_ACCELERATOR=cuda DS_ENABLE_NINJA=1 TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0" DS_BUILD_OPS=1 DS_BUILD_SPARSE_ATTN=0 DS_BUILD_FP_QUANTIZER=0 DS_BUILD_CUTLASS_OPS=0 DS_BUILD_GDS=0 DS_BUILD_RAGGED_DEVICE_OPS=0 DS_BUILD_EVOFORMER_ATTN=0 DS_BUILD_DEEP_COMPILE=0 pip3 install .
+            DS_ACCELERATOR=cuda DS_ENABLE_NINJA=1 TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0" DS_BUILD_OPS=1 DS_BUILD_SPARSE_ATTN=0 DS_BUILD_FP_QUANTIZER=0 DS_BUILD_CUTLASS_OPS=0 DS_BUILD_GDS=0 DS_BUILD_RAGGED_DEVICE_OPS=0 DS_BUILD_EVOFORMER_ATTN=0 DS_BUILD_DEEP_COMPILE=0 pip3 install . --no-build-isolation
         - name: DS Report
           run: |
              ds_report

.github/workflows/nv-torch-nightly-v100.yml

@@ -43,7 +43,7 @@ jobs:
 
       - name: Install deepspeed
         run: |
-          pip install .[dev,1bit,autotuning]
+          pip install .[dev,1bit,autotuning] --no-build-isolation
           ds_report
 
       - name: Python environment

deepspeed/git_version_info.py

@@ -4,7 +4,7 @@
 # DeepSpeed Team
 
 try:
-    #  This is populated by setup.py
+    #  This is populated by setup.py/pyproject.toml
     from .git_version_info_installed import *  # noqa: F401 # type: ignore
 except ModuleNotFoundError:
     import os

pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = [
+    "build",
+    "setuptools>=64",
+    "torch",
+    "wheel"
+]
+# Use legacy backend to import local packages in setup.py
+build-backend = "setuptools.build_meta:__legacy__"