Update pymaro build wheel logic (#587)

* add libffi-dev, rust toolchain to linux build wheel

* try updating build wheel

* update python version in build wheel script

* ignore install requirements in build wheel workflow

* fix python versions in build wheel

* Update build wheel action (#586)

* avoid install unused package while building wheels

* move source package to windows, as it will cause permission issue on linux

* fix manylinux python version

* remove duplicated definition

---------

Co-authored-by: Jinyu Wang <[email protected]>
Co-authored-by: Chaos Yu <[email protected]>

.github/workflows/build_wheel.yml

@@ -41,26 +41,29 @@ jobs:
       run: |
         python setup.py bdist_wheel
 
+    - name: Build source package on Windows
+      if: runner.os == 'Windows' && matrix.python-version == '3.7'
+      run: |
+        python setup.py sdist
+
     - name: Build manylinux wheel
       if: runner.os == 'Linux' && matrix.python-version == '3.7'
       uses: RalfG/[email protected]_x86_64
+      env:
+        GITHUB_BUILD_ACTION: True
       with:
-        python-versions: 'cp37-cp37m cp38-cp38m cp39-cp39m'
+        python-versions: 'cp37-cp37m cp38-cp38 cp39-cp39'
         build-requirements: 'numpy'
-        pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder
+        system-packages: 'libffi-devel'
         pre-build-command: 'export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH'
+        pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder
 
     - name: Move valid packages to dist folder for manylinux
       if: runner.os == 'Linux' && matrix.python-version == '3.7'
       run: |
         mkdir -p dist
         cp wheelhouse/pymaro-*-manylinux*.whl dist
 
-    - name: Build source package on linux
-      if: runner.os == 'Linux' && matrix.python-version == '3.7'
-      run: |
-        python setup.py sdist
-
     - name: Upload linux wheel to artifact
       uses: actions/upload-artifact@v2
       with:

scripts/build_maro.bat

@@ -1,10 +1,10 @@
-@ECHO OFF
-
-rem script to build maro locally on Windows, usually for development
-
-chdir "%~dp0.."
-
-rem compile cython files
-call scripts\compile_cython.bat
-
-python setup.py build_ext -i
+@ECHO OFF
+
+rem script to build maro locally on Windows, usually for development
+
+chdir "%~dp0.."
+
+rem compile cython files
+call scripts\compile_cython.bat
+
+python setup.py build_ext -i

scripts/build_maro.sh

@@ -10,4 +10,4 @@ fi
 # compile cython files first
 bash ./scripts/compile_cython.sh
 
-python setup.py build_ext -i
+python setup.py build_ext -i

scripts/build_sdist.bat

@@ -1,3 +1,3 @@
-chdir "%~dp0.."
-
-python setup.py sdist
+chdir "%~dp0.."
+
+python setup.py sdist

scripts/build_sdist.sh

@@ -10,4 +10,4 @@ fi
 
 bash ./scripts/compile_cython.sh
 
-python setup.py sdist
+python setup.py sdist

scripts/build_wheel.bat

@@ -1,14 +1,14 @@
-
-rem script to build wheel package on Windows
-rem NOTE: Before building the wheels, please make sure you have setup-up the environment.
-rem for python 3.6/3.7 we need vs++14
-
-chdir "%~dp0.."
-
-call scripts\compile_cython.bat
-
-pip install -r maro/requirements.build.txt
-pip install wheel
-pip install --upgrade setuptools
-
-python setup.py bdist_wheel
+
+rem script to build wheel package on Windows
+rem NOTE: Before building the wheels, please make sure you have setup-up the environment.
+rem for python 3.6/3.7 we need vs++14
+
+chdir "%~dp0.."
+
+call scripts\compile_cython.bat
+
+pip install -r maro/requirements.build.txt
+pip install wheel
+pip install --upgrade setuptools
+
+python setup.py bdist_wheel

scripts/build_wheel.sh

@@ -8,7 +8,7 @@ set -e -x
 cd /maro
 
 # Compile wheels
-for PYVER in 6 7; do
+for PYVER in 7 8 9; do
     PYBIN="/opt/python/cp3${PYVER}-cp3${PYVER}m/bin"
 
     "${PYBIN}/pip" install -r maro/requirements.build.txt

scripts/install_maro.bat

@@ -1,17 +1,17 @@
-@ECHO OFF
-
-rem Script to install MARO in editable mode on Windows,
-rem usually for development.
-
-chdir "%~dp0.."
-
-rem Install dependencies.
-pip install -r .\maro\requirements.build.txt
-
-rem Compile cython files.
-call .\scripts\compile_cython.bat
-
-call .\scripts\install_torch.bat
-
-rem Install MARO in editable mode.
-pip install -e .
+@ECHO OFF
+
+rem Script to install MARO in editable mode on Windows,
+rem usually for development.
+
+chdir "%~dp0.."
+
+rem Install dependencies.
+pip install -r .\maro\requirements.build.txt
+
+rem Compile cython files.
+call .\scripts\compile_cython.bat
+
+call .\scripts\install_torch.bat
+
+rem Install MARO in editable mode.
+pip install -e .

scripts/install_torch.bat

@@ -1 +1 @@
-pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html
+pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html

scripts/run_tests.bat

@@ -1,17 +1,17 @@
-rem script to run all the test script under tests folder which the file name match test_xxxx.py
-
-chdir "%~dp0.."
-
-set "PYTHONPAH=."
-
-call scripts/build_maro.bat
-
-rem install requirements
-
-pip install -r ./tests/requirements.test.txt
-
-rem show coverage
-
-coverage run --rcfile=./tests/.coveragerc
-
-coverage report --rcfile=./tests/.coveragerc
+rem script to run all the test script under tests folder which the file name match test_xxxx.py
+
+chdir "%~dp0.."
+
+set "PYTHONPAH=."
+
+call scripts/build_maro.bat
+
+rem install requirements
+
+pip install -r ./tests/requirements.test.txt
+
+rem show coverage
+
+coverage run --rcfile=./tests/.coveragerc
+
+coverage report --rcfile=./tests/.coveragerc

scripts/run_tests.sh

@@ -17,4 +17,4 @@ pip install -r ./tests/requirements.test.txt
 
 coverage run --rcfile=./tests/.coveragerc
 
-coverage report --rcfile=./tests/.coveragerc
+coverage report --rcfile=./tests/.coveragerc

scripts/show_test_coverage_html.bat

@@ -1,19 +1,19 @@
-
-chdir "%~dp0.."
-
-rem remove old htmlcov
-@RD /S /Q htmlcov
-
-rem generate html
-
-coverage html
-
-rem host html
-
-cd htmlcov
-
-REM python -m http.server 8888
-
-start python -m http.server 8888
-
-start "" "http://localhost:8888"
+
+chdir "%~dp0.."
+
+rem remove old htmlcov
+@RD /S /Q htmlcov
+
+rem generate html
+
+coverage html
+
+rem host html
+
+cd htmlcov
+
+REM python -m http.server 8888
+
+start python -m http.server 8888
+
+start "" "http://localhost:8888"

setup.py

@@ -25,7 +25,6 @@
 # Set environment variable to skip deployment process of MARO
 os.environ["SKIP_DEPLOYMENT"] = "TRUE"
 
-
 # root path to backend
 BASE_SRC_PATH = "./maro/backends"
 # backend module name
@@ -51,7 +50,6 @@
     ),
 )
 
-
 include_dirs.append(numpy.get_include())
 
 extensions.append(
@@ -84,9 +82,31 @@
     ),
 )
 
-specific_requires = []
+# It is not necessary to install these packages when using manylinux action, as we only need numpy to build wheels
+# NOTE: install following package will cause build error in current(2023-03-14) manylinux image
+if "GITHUB_BUILD_ACTION" in os.environ:
+    install_requires = []
+else:
+    install_requires = [
+        # TODO: use a helper function to collect these
+        "holidays>=0.10.3",
+        "numpy>=1.19.5",
+        "pandas>=0.25.3",
+        "paramiko>=2.9.2",
+        "ptvsd>=4.3.2",
+        "python_dateutil>=2.8.1",
+        "PyYAML>=5.4.1",
+        "pyzmq>=19.0.2",
+        "redis>=3.5.3",
+        "requests>=2.25.1",
+        "scipy>=1.7.0",
+        "tabulate>=0.8.5",
+        "torch>=1.6.0, <1.14.0",
+        "tornado>=6.1",
+    ]
+
 if sys.version.startswith("3.6"):
-    specific_requires.append("dataclasses>=0.5")
+    install_requires.append("dataclasses>=0.5")
 
 readme = io.open("./maro/README.rst", encoding="utf-8").read()
 
@@ -133,24 +153,7 @@
     setup_requires=[
         "numpy<1.20.0",
     ],
-    install_requires=[
-        # TODO: use a helper function to collect these
-        "holidays>=0.10.3",
-        "numpy>=1.19.5",
-        "pandas>=0.25.3",
-        "paramiko>=2.9.2",
-        "ptvsd>=4.3.2",
-        "python_dateutil>=2.8.1",
-        "PyYAML>=5.4.1",
-        "pyzmq>=19.0.2",
-        "redis>=3.5.3",
-        "requests>=2.25.1",
-        "scipy>=1.7.0",
-        "tabulate>=0.8.5",
-        "torch>=1.6.0, <1.14.0",
-        "tornado>=6.1",
-    ]
-    + specific_requires,
+    install_requires=install_requires,
     entry_points={
         "console_scripts": [
             "maro=maro.cli.maro:main",