From 3ae696aa763c7b25c2587c48441e47a4d0926a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 22 May 2024 14:44:31 +0200 Subject: [PATCH] Extend CI to test with onnxruntime==1.18.0 (#692) * Extend CI to test with onnxruntime==1.18.0 Signed-off-by: Xavier Dupre * style Signed-off-by: Xavier Dupre * restore trigger on main Signed-off-by: Xavier Dupre * update CI Signed-off-by: Xavier Dupre * doc Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * remove unnecessary steps Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * fix ci Signed-off-by: Xavier Dupre * fix ci Signed-off-by: Xavier Dupre * fix issue with python 3.12 Signed-off-by: Xavier Dupre * fix Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * pin version for scikit-learn Signed-off-by: Xavier Dupre * skl Signed-off-by: Xavier Dupre * mac Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * precision Signed-off-by: Xavier Dupre * fix ci Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre * ci Signed-off-by: Xavier Dupre --------- Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-conda-CI.yml | 203 ------------------ .azure-pipelines/win32-conda-CI.yml | 192 ----------------- .github/workflows/ci.yml | 144 +++++++++++++ README.md | 10 +- onnxmltools/convert/README.md | 2 +- .../tree_ensemble_common.py | 6 +- .../xgboost/operator_converters/XGBoost.py | 8 +- onnxmltools/proto/__init__.py | 8 +- onnxmltools/utils/tests_helper.py | 16 +- requirements-dev.txt | 4 +- tests/xgboost/test_xgboost_converters.py | 2 +- 11 files changed, 174 insertions(+), 421 deletions(-) delete mode 100644 .azure-pipelines/linux-conda-CI.yml delete mode 100644 .azure-pipelines/win32-conda-CI.yml create mode 100644 .github/workflows/ci.yml diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml deleted file mode 100644 index f1d953828..000000000 --- a/.azure-pipelines/linux-conda-CI.yml +++ /dev/null @@ -1,203 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - timeoutInMinutes: 30 - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - - Python311-1160-RT1171-xgb2-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.17.1' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=2' - numpy.version: '' - scipy.version: '' - - Python311-1150-RT1163-xgb2-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=2' - numpy.version: '' - scipy.version: '' - - Python311-1150-RT1163-xgb175-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python311-1141-RT1162-xgb175-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python310-1141-RT1151-xgb175: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - lightgbm.version: '<4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python310-1141-RT1140-xgb175: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.14.0' - COREML_PATH: NONE - lightgbm.version: '<4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python39-1141-RT1151-xgb175-scipy180: - python.version: '3.9' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '==1.8.0' - - - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - script: | - python -m pip install --upgrade pip - pip install $(ONNX_PATH) $(ONNXRT_PATH) cython - pip install -r requirements.txt - displayName: 'Install dependencies' - - - script: | - pip install -r requirements-dev.txt - displayName: 'Install dependencies-dev' - - - script: | - python -m pip install --upgrade pip - pip install "xgboost$(xgboost.version)" - pip install "lightgbm$(lightgbm.version)" - pip install $(ONNX_PATH) - pip install $(ONNXRT_PATH) - pip install "numpy$(numpy.version)" - pip install "scipy$(scipy.version)" - displayName: 'Install xgboost, onnxruntime' - - - script: | - python -m pip install coloredlogs flatbuffers packaging sympy numpy protobuf - python -m pip install $(ONNXRT_PATH) - displayName: 'Install onnxruntime' - - - script: | - pip install -e . - displayName: 'local installation' - - - script: | - export PYTHONPATH=. - python -c "import onnxruntime;print('onnx:',onnx.__version__)" - python -c "import onnxconverter_common;print('cc:',onnxconverter_common.__version__)" - python -c "import onnx;print('onnx:',onnx.__version__)" - python -c "import onnxruntime;print('ort:',onnxruntime.__version__)" - python -c "import xgboost;print('xgboost:',xgboost.__version__)" - python -c "import lightgbm;print('lightgbm:',lightgbm.__version__)" - displayName: 'version' - - - script: | - export PYTHONPATH=. - pytest tests/baseline --durations=0 - displayName: 'pytest - baseline' - - - script: | - export PYTHONPATH=. - pytest tests/catboost --durations=0 - displayName: 'pytest - catboost' - - - script: | - export PYTHONPATH=. - pytest tests/lightgbm --durations=0 - displayName: 'pytest - lightgbm' - - - script: | - export PYTHONPATH=. - pytest tests/sparkml --durations=0 - displayName: 'pytest - sparkml' - - - script: | - export PYTHONPATH=. - pytest tests/utils --durations=0 - displayName: 'pytest - utils' - - - script: | - export PYTHONPATH=. - pytest tests/xgboost --durations=0 - displayName: 'pytest - xgboost' - - - script: | - export PYTHONPATH=. - pip install h2o - pytest tests/h2o --durations=0 - displayName: 'pytest - h2o' - - - script: | - export PYTHONPATH=. - pytest tests/svmlib --durations=0 - displayName: 'pytest - svmlib' - - - script: | - pip install torch --extra-index-url https://download.pytorch.org/whl/cpu - pip install hummingbird-ml --no-deps - displayName: 'Install hummingbird-ml' - - - script: | - export PYTHONPATH=. - pytest tests/hummingbirdml --durations=0 - displayName: 'pytest - hummingbirdml' - - - script: | - if [ '$(COREML_PATH)' == 'NONE' ] - then - echo "required version of coremltools does not work on python 3.10" - else - export PYTHONPATH=. - pip install $(COREML_PATH) - pytest tests/coreml --durations=0 - fi - displayName: 'pytest - coreml [$(COREML_PATH)]' - # condition: ne('$(COREML_PATH)', 'NONE') - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml deleted file mode 100644 index 4fbe704d9..000000000 --- a/.azure-pipelines/win32-conda-CI.yml +++ /dev/null @@ -1,192 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - timeoutInMinutes: 30 - pool: - vmImage: 'windows-latest' - strategy: - matrix: - - Python311-1160-RT1171: - python.version: '3.11' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.17.1' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '>=2.0' - - Python311-1150-RT1163: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==2.0.2' - - Python311-1150-RT1162: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==2.0.2' - - Python311-1141-RT1162: - python.version: '3.11' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python310-1141-RT1151: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python310-1141-RT1140: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: onnxruntime==1.14.0 - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python39-1141-RT1140: - python.version: '3.9' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: onnxruntime==1.14.0 - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - maxParallel: 3 - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf scikit-learn scipy cython - displayName: Create Anaconda environment - - - script: | - call activate py$(python.version) - python -m pip install --upgrade pip numpy - echo Test numpy installation... && python -c "import numpy" - python -m pip install -r requirements.txt - displayName: 'Install dependencies (1)' - - - script: | - call activate py$(python.version) - python -m pip install -r requirements-dev.txt - displayName: 'Install dependencies-dev' - - - script: | - call activate py$(python.version) - python -m pip install --upgrade scikit-learn - python -m pip install --upgrade lightgbm - python -m pip install "xgboost$(xgboost.version)" - displayName: 'Install scikit-learn' - - - script: | - call activate py$(python.version) - python -m pip install %ONNX_PATH% - python -m pip install %ONNXRT_PATH% - python -m pip install "numpy$(numpy.version)" - displayName: 'Install onnxruntime' - - - script: | - call activate py$(python.version) - python -m pip install coloredlogs flatbuffers packaging sympy numpy protobuf - python -m pip install $(ONNXRT_PATH) - displayName: 'Install ort-nightly' - - - script: | - call activate py$(python.version) - python -m pip install -e . - export PYTHONPATH=. - python -c "import onnxconverter_common;print(onnxconverter_common.__version__)" - python -c "import onnx;print(onnx.__version__)" - python -c "import onnxruntime;print(onnxruntime.__version__)" - python -c "import xgboost;print(xgboost.__version__)" - python -c "import lightgbm;print(lightgbm.__version__)" - displayName: 'version' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/baseline --durations=0 - displayName: 'pytest baseline' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/catboost --durations=0 - displayName: 'pytest catboost' - - - script: | - call activate py$(python.version) - set PYTHONPATH=. - if "$(COREML_PATH)" neq "NONE" python -m pip install %COREML_PATH% - if "$(COREML_PATH)" neq "NONE" python -m pytest tests/coreml --durations=0 - displayName: 'pytest coreml - [$(COREML_PATH)]' - #condition: ne('$(COREML_PATH)', 'NONE') - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/lightgbm --durations=0 - displayName: 'pytest lightgbm' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/sparkml --durations=0 - displayName: 'pytest sparkml' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/svmlib --durations=0 - displayName: 'pytest svmlib' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/utils --durations=0 - displayName: 'pytest utils' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/xgboost --durations=0 - displayName: 'pytest xgboost' - - - script: | - call activate py$(python.version) - python -m pip install torch - python -m pip install hummingbird-ml --no-deps - displayName: 'Install hummingbird-ml' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/hummingbirdml --durations=0 - displayName: 'pytest hummingbirdml' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..0b51164f5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,144 @@ +name: CI +on: [push, pull_request] +jobs: + run: + name: ${{ matrix.os }} py==${{ matrix.python_version }} - sklearn${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python_version: ['3.12', '3.11', '3.10', '3.9'] + include: + - python_version: '3.12' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.16.0' + onnxrt_version: 'onnxruntime==1.18.0' + sklearn_version: '==1.4.2' + lgbm_version: ">=4" + xgboost_version: ">=2" + - python_version: '3.11' + documentation: 1 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.4.2' + lgbm_version: ">=4" + xgboost_version: ">=2" + - python_version: '3.10' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.14.1' + onnxrt_version: 'onnxruntime==1.16.3' + sklearn_version: '==1.4.2' + lgbm_version: "<4" + xgboost_version: "<2" + - python_version: '3.9' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.13.0' + onnxrt_version: 'onnxruntime==1.14.1' + sklearn_version: '==1.4.2' + lgbm_version: "<4" + xgboost_version: "<2" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" "xgboost${{ matrix.xgboost_version }}" + + - name: Install lightgbm + if: matrix.os != 'macos-latest' + run: pip install "lightgbm${{ matrix.lgbm_version }}" + + - name: Install + run: pip install -e . + + - name: versions + run: | + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from xgboost import __version__;print('xgboost', __version__)" + python -c "from catboost import __version__;print('catboost', __version__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" + + - name: versions lightgbm + if: matrix.os != 'macos-latest' + run: | + python -c "from lightgbm import __version__;print('lightgbm', __version__)" + + - name: Run tests baseline + run: pytest --maxfail=10 --durations=10 tests/baseline + + - name: Run tests utils + run: pytest --maxfail=10 --durations=10 tests/utils + + - name: Run tests catboost + run: pytest --maxfail=10 --durations=10 tests/catboost + + - name: Run tests lightgbm + if: matrix.os != 'macos-latest' + run: pytest --maxfail=10 --durations=10 tests/lightgbm + + - name: Run tests xgboost + run: pytest --maxfail=10 --durations=10 tests/xgboost + + - name: Run tests svmlib + run: pytest --maxfail=10 --durations=10 tests/svmlib + + - name: Run tests h2o + if: matrix.os == 'ubuntu-latest' + run: | + pip install h2o + pytest --maxfail=10 --durations=10 tests/h2o + + - name: Run tests pysparkml + if: matrix.os == 'ubuntu-latest' && matrix.python_version != '3.12' + run: pytest --maxfail=10 --durations=10 tests/sparkml + + - name: Run tests hummingbirdml + if: matrix.os != 'macos-latest' + run: | + pip install torch --extra-index-url https://download.pytorch.org/whl/cpu + pip install hummingbird-ml --no-deps + pytest --maxfail=10 --durations=10 tests/hummingbirdml + + - name: Run tests baseline + run: pytest --maxfail=10 --durations=10 tests/baseline + + - name: build + run: pip wheel . + + - uses: actions/upload-artifact@v4 + with: + path: ./dist/** diff --git a/README.md b/README.md index 2caba11f3..6f2965d9c 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,9 @@ # -![ONNXMLTools_logo_main](docs/ONNXMLTools_logo_main.png) +![ONNXMLTools_logo_main](https://github.com/onnx/onnxmltools/blob/main/docs/ONNXMLTools_logo_main.png) -| Linux | Windows | -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [![Build Status](https://dev.azure.com/onnxmltools/onnxmltools/_apis/build/status/onnxmltools-linux-conda-ci?branchName=master)](https://dev.azure.com/onnxmltools/onnxmltools/_build/latest?definitionId=3?branchName=master) | [![Build Status](https://dev.azure.com/onnxmltools/onnxmltools/_apis/build/status/onnxmltools-win32-conda-ci?branchName=master)](https://dev.azure.com/onnxmltools/onnxmltools/_build/latest?definitionId=3?branchName=master) | +[![CI](https://github.com/onnx/onnxmltools/actions/workflows/ci.yml/badge.svg)](https://github.com/onnx/onnxmltools/actions/workflows/ci.yml) ## Introduction @@ -58,7 +56,7 @@ ONNXMLTools is tested with Python **3.7+**. # Examples -If you want the converted ONNX model to be compatible with a certain ONNX version, please specify the target_opset parameter upon invoking the convert function. The following Keras model conversion example demonstrates this below. You can identify the mapping from ONNX Operator Sets (referred to as opsets) to ONNX releases in the [versioning documentation](https://github.com/onnx/onnx/blob/master/docs/Versioning.md#released-versions). +If you want the converted ONNX model to be compatible with a certain ONNX version, please specify the target_opset parameter upon invoking the convert function. The following Keras model conversion example demonstrates this below. You can identify the mapping from ONNX Operator Sets (referred to as opsets) to ONNX releases in the [versioning documentation](https://github.com/onnx/onnx/blob/main/docs/Versioning.md#released-versions). ## Keras to ONNX Conversion @@ -142,7 +140,7 @@ opset_version = onnx_model.opset_import[0].version If the result from checking your ONNX model's opset is smaller than the `target_opset` number you specified in the onnxmltools.convert function, be assured that this is likely intended behavior. The ONNXMLTools converter works by converting each operator to the ONNX format individually and finding the corresponding opset version that it was most recently updated in. Once all of the operators are converted, the resultant ONNX model has the maximal opset version of all of its operators. -To illustrate this concretely, let's consider a model with two operators, Abs and Add. As of December 2018, [Abs](https://github.com/onnx/onnx/blob/master/docs/Operators.md#abs) was most recently updated in opset 6, and [Add](https://github.com/onnx/onnx/blob/master/docs/Operators.md#add) was most recently updated in opset 7. Therefore, the converted ONNX model's opset will always be 7, even if you request `target_opset=8`. The converter behavior was defined this way to ensure backwards compatibility. +To illustrate this concretely, let's consider a model with two operators, Abs and Add. As of December 2018, [Abs](https://github.com/onnx/onnx/blob/main/docs/Operators.md#abs) was most recently updated in opset 6, and [Add](https://github.com/onnx/onnx/blob/main/docs/Operators.md#add) was most recently updated in opset 7. Therefore, the converted ONNX model's opset will always be 7, even if you request `target_opset=8`. The converter behavior was defined this way to ensure backwards compatibility. Documentation for the [ONNX Model format](https://github.com/onnx/onnx) and more examples for converting models from different frameworks can be found in the [ONNX tutorials](https://github.com/onnx/tutorials) repository. diff --git a/onnxmltools/convert/README.md b/onnxmltools/convert/README.md index ebc828cb4..ae77ab108 100644 --- a/onnxmltools/convert/README.md +++ b/onnxmltools/convert/README.md @@ -81,7 +81,7 @@ The shape mapping from Core ML to our IR obeys the following rules. Notice that the compiler can overwrite those rules at some stages like shape inference. An example is the label shape of a classifier. One may expect that its shape is `[1, 1].` Nevertheless, our shape inference may change it to `[1]`. The major reason is that the current definition of ZipMap, the operator used to generate the predicted probabilities, does not support batch size greater than one. Core ML's batch size, `N-axis`, is ignored because it is not related to graph structures. In fact, ONNX's batch size is rather equivalent to sequence axis in Core ML. By default, we use `N=1` for traditional machine learning models and `N='None'` for neural networks. To overwrite our default types, user can provide `initial_types` when calling `convert(...)` defined in `onnxmltools.convert.coreml.convert.py`. All Core ML's shape calculations are derived from [this document](https://apple.github.io/coremltools/coremlspecification/index.html) specifically for our type system. -Some more details about Core ML neural network operator can be found at this [page](https://github.com/apple/coremltools/blob/master/mlmodel/format/NeuralNetwork.proto) +Some more details about Core ML neural network operator can be found at this [page](https://github.com/apple/coremltools/blob/main/mlmodel/format/NeuralNetwork.proto) For scikit-learn, user may need to specify the input types for their models. In general, we expect `[1, C]` if the input is feature vector. diff --git a/onnxmltools/convert/sparkml/operator_converters/tree_ensemble_common.py b/onnxmltools/convert/sparkml/operator_converters/tree_ensemble_common.py index a31a7dc4d..b4c9192e2 100644 --- a/onnxmltools/convert/sparkml/operator_converters/tree_ensemble_common.py +++ b/onnxmltools/convert/sparkml/operator_converters/tree_ensemble_common.py @@ -28,7 +28,11 @@ def sparkml_tree_dataset_to_sklearn(tree_df, is_classifier): if isinstance(item, dict): try: feature.append(item["featureIndex"]) - threshold.append(item["leftCategoriesOrThreshold"][0] if len(item["leftCategoriesOrThreshold"]) >= 1 else -1.0) + threshold.append( + item["leftCategoriesOrThreshold"][0] + if len(item["leftCategoriesOrThreshold"]) >= 1 + else -1.0 + ) except KeyError: raise RuntimeError(f"Unable to process {item}.") else: diff --git a/onnxmltools/convert/xgboost/operator_converters/XGBoost.py b/onnxmltools/convert/xgboost/operator_converters/XGBoost.py index d241f1868..81185428b 100644 --- a/onnxmltools/convert/xgboost/operator_converters/XGBoost.py +++ b/onnxmltools/convert/xgboost/operator_converters/XGBoost.py @@ -120,10 +120,10 @@ def _add_node( # Split condition for sklearn # * if X_ptr[X_sample_stride * i + X_fx_stride * node.feature] <= node.threshold: - # * https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/tree/_tree.pyx#L946 + # * https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/tree/_tree.pyx#L946 # Split condition for xgboost # * if (fvalue < split_value) - # * https://github.com/dmlc/xgboost/blob/master/include/xgboost/tree_model.h#L804 + # * https://github.com/dmlc/xgboost/blob/main/include/xgboost/tree_model.h#L804 attr_pairs["nodes_treeids"].append(tree_id) attr_pairs["nodes_nodeids"].append(node_id) @@ -332,7 +332,7 @@ def convert(scope, operator, container): if ncl <= 1: ncl = 2 if objective != "binary:hinge": - # See https://github.com/dmlc/xgboost/blob/master/src/common/math.h#L23. + # See https://github.com/dmlc/xgboost/blob/main/src/common/math.h#L23. attr_pairs["post_transform"] = "LOGISTIC" attr_pairs["class_ids"] = [0 for v in attr_pairs["class_treeids"]] if js_trees[0].get("leaf", None) == 0: @@ -344,7 +344,7 @@ def convert(scope, operator, container): else: attr_pairs["base_values"] = [base_score] else: - # See https://github.com/dmlc/xgboost/blob/master/src/common/math.h#L35. + # See https://github.com/dmlc/xgboost/blob/main/src/common/math.h#L35. attr_pairs["post_transform"] = "SOFTMAX" attr_pairs["base_values"] = [base_score for n in range(ncl)] attr_pairs["class_ids"] = [v % ncl for v in attr_pairs["class_treeids"]] diff --git a/onnxmltools/proto/__init__.py b/onnxmltools/proto/__init__.py index a6bd5fafb..e84b6419c 100644 --- a/onnxmltools/proto/__init__.py +++ b/onnxmltools/proto/__init__.py @@ -15,13 +15,9 @@ def _check_onnx_version(): - import pkg_resources + from onnx import __version__ - min_required_version = pkg_resources.parse_version("1.0.1") - current_version = pkg_resources.get_distribution("onnx").parsed_version - assert ( - current_version >= min_required_version - ), "ONNXMLTools requires ONNX version 1.0.1 or a newer one" + return not __version__.startswith("0.") _check_onnx_version() diff --git a/onnxmltools/utils/tests_helper.py b/onnxmltools/utils/tests_helper.py index 7b5fd1d49..78d8eb81d 100644 --- a/onnxmltools/utils/tests_helper.py +++ b/onnxmltools/utils/tests_helper.py @@ -88,10 +88,16 @@ def dump_data_and_model( os.makedirs(folder) if hasattr(model, "predict"): - import lightgbm - import xgboost - - if isinstance(model, lightgbm.Booster): + try: + import lightgbm + except ImportError: + lightgbm = None + try: + import xgboost + except ImportError: + xgboost = None + + if lightgbm is not None and isinstance(model, lightgbm.Booster): # LightGBM Booster model_dict = model.dump_model() if model_dict["objective"].startswith("binary"): @@ -105,7 +111,7 @@ def dump_data_and_model( prediction = [score.argmax(axis=1), score] else: prediction = [model.predict(data)] - elif isinstance(model, xgboost.Booster): + elif xgboost is not None and isinstance(model, xgboost.Booster): # XGBoost Booster from ..convert.xgboost._parse import _get_attributes from xgboost import DMatrix diff --git a/requirements-dev.txt b/requirements-dev.txt index 1ba960b8e..1fe484ec6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,13 +3,13 @@ catboost cython dill libsvm -lightgbm +lightgbm; sys_platform != 'darwin' mleap numpy openpyxl onnxconverter-common pandas -pyspark +pyspark; sys_platform == 'linux' pytest pytest-cov pytest-spark diff --git a/tests/xgboost/test_xgboost_converters.py b/tests/xgboost/test_xgboost_converters.py index ab33e845a..c364257a4 100644 --- a/tests/xgboost/test_xgboost_converters.py +++ b/tests/xgboost/test_xgboost_converters.py @@ -173,7 +173,7 @@ def test_xgb_classifier_reglog(self): ) dump_data_and_model( - x_test, xgb, conv_model, basename="SklearnXGBClassifierRegLog" + x_test, xgb, conv_model, basename="SklearnXGBClassifierRegLog-Dec4" ) def test_xgb_classifier_multi_discrete_int_labels(self):