From a105247903398c9c2e26906af447d90cb8f380b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 3 Apr 2024 11:51:41 +0200 Subject: [PATCH] Upgrade CI to onnx==1.16.0 (#689) * upgrade to onnx==1.16.0 Signed-off-by: Xavier Dupre * doc Signed-off-by: Xavier Dupre * black Signed-off-by: Xavier Dupre * black Signed-off-by: Xavier Dupre * fix opset Signed-off-by: Xavier Dupre --------- Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-conda-CI.yml | 6 +-- .azure-pipelines/win32-conda-CI.yml | 22 ++++----- .github/workflows/black-ruff.yml | 2 +- CHANGELOGS.md | 2 + .../operator_converters/TreeEnsemble.py | 18 +++---- .../convert/sparkml/ops_input_output.py | 48 ++++++++++++------- .../xgboost/shape_calculators/Classifier.py | 2 +- pyproject.toml | 4 +- tests/xgboost/test_xgboost_issues.py | 4 +- 9 files changed, 62 insertions(+), 46 deletions(-) diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index 8388d6ee..f1d95382 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -15,10 +15,10 @@ jobs: strategy: matrix: - Python311-1150-RT117-xgb2-lgbm40: + Python311-1160-RT1171-xgb2-lgbm40: python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.17' + ONNX_PATH: 'onnx==1.16.0' + ONNXRT_PATH: 'onnxruntime==1.17.1' COREML_PATH: NONE lightgbm.version: '>=4.0' xgboost.version: '>=2' diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index c87f9491..4fbe704d 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -15,13 +15,13 @@ jobs: strategy: matrix: - Python311-1150-RT117: + Python311-1160-RT1171: python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.17' + ONNX_PATH: 'onnx==1.16.0' + ONNXRT_PATH: 'onnxruntime==1.17.1' COREML_PATH: NONE numpy.version: '' - xgboost.version: '2.0.2' + xgboost.version: '>=2.0' Python311-1150-RT1163: python.version: '3.11' @@ -29,7 +29,7 @@ jobs: ONNXRT_PATH: 'onnxruntime==1.16.3' COREML_PATH: NONE numpy.version: '' - xgboost.version: '2.0.2' + xgboost.version: '==2.0.2' Python311-1150-RT1162: python.version: '3.11' @@ -37,7 +37,7 @@ jobs: ONNXRT_PATH: 'onnxruntime==1.16.2' COREML_PATH: NONE numpy.version: '' - xgboost.version: '2.0.2' + xgboost.version: '==2.0.2' Python311-1141-RT1162: python.version: '3.11' @@ -45,7 +45,7 @@ jobs: ONNXRT_PATH: 'onnxruntime==1.16.2' COREML_PATH: NONE numpy.version: '' - xgboost.version: '1.7.5' + xgboost.version: '==1.7.5' Python310-1141-RT1151: python.version: '3.10' @@ -53,7 +53,7 @@ jobs: ONNXRT_PATH: 'onnxruntime==1.15.1' COREML_PATH: NONE numpy.version: '' - xgboost.version: '1.7.5' + xgboost.version: '==1.7.5' Python310-1141-RT1140: python.version: '3.10' @@ -61,7 +61,7 @@ jobs: ONNXRT_PATH: onnxruntime==1.14.0 COREML_PATH: NONE numpy.version: '' - xgboost.version: '1.7.5' + xgboost.version: '==1.7.5' Python39-1141-RT1140: python.version: '3.9' @@ -69,7 +69,7 @@ jobs: ONNXRT_PATH: onnxruntime==1.14.0 COREML_PATH: NONE numpy.version: '' - xgboost.version: '1.7.5' + xgboost.version: '==1.7.5' maxParallel: 3 @@ -96,7 +96,7 @@ jobs: call activate py$(python.version) python -m pip install --upgrade scikit-learn python -m pip install --upgrade lightgbm - python -m pip install "xgboost==$(xgboost.version)" + python -m pip install "xgboost$(xgboost.version)" displayName: 'Install scikit-learn' - script: | diff --git a/.github/workflows/black-ruff.yml b/.github/workflows/black-ruff.yml index 09da3fc3..dc846df3 100644 --- a/.github/workflows/black-ruff.yml +++ b/.github/workflows/black-ruff.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: psf/black@193ee766ca496871f93621d6b58d57a6564ff81b # stable 23.7.0 + - uses: psf/black@e42f1ffbe159018c8e887f775c0fd7b51b3cf787 # stable 24.3.0 with: options: "--diff --check" src: "." diff --git a/CHANGELOGS.md b/CHANGELOGS.md index 6598aac4..2920b296 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -2,6 +2,8 @@ ## 1.13.0 (development) +* Handle issue with binary classifier setting output to [N,1] vs [N,2], + [#681](https://github.com/onnx/onnxmltools/pull/681) * Add missing dependency onnxconverter_common, fix multi regression with xgboost, [#679](https://github.com/onnx/onnxmltools/pull/679), fixes issues [No module named 'onnxconverter_common'](https://github.com/onnx/onnxmltools/issues/673), diff --git a/onnxmltools/convert/coreml/operator_converters/TreeEnsemble.py b/onnxmltools/convert/coreml/operator_converters/TreeEnsemble.py index db52cc01..6dda3fbf 100644 --- a/onnxmltools/convert/coreml/operator_converters/TreeEnsemble.py +++ b/onnxmltools/convert/coreml/operator_converters/TreeEnsemble.py @@ -45,9 +45,9 @@ def convert_tree_ensemble_model(scope, operator, container): op_type = "TreeEnsembleClassifier" prefix = "class" nodes = raw_model.treeEnsembleClassifier.treeEnsemble.nodes - attrs[ - "base_values" - ] = raw_model.treeEnsembleClassifier.treeEnsemble.basePredictionValue + attrs["base_values"] = ( + raw_model.treeEnsembleClassifier.treeEnsemble.basePredictionValue + ) attrs["post_transform"] = get_onnx_tree_post_transform( raw_model.treeEnsembleClassifier.postEvaluationTransform ) @@ -72,12 +72,12 @@ def convert_tree_ensemble_model(scope, operator, container): op_type = "TreeEnsembleRegressor" prefix = "target" nodes = raw_model.treeEnsembleRegressor.treeEnsemble.nodes - attrs[ - "base_values" - ] = raw_model.treeEnsembleRegressor.treeEnsemble.basePredictionValue - attrs[ - "n_targets" - ] = raw_model.treeEnsembleRegressor.treeEnsemble.numPredictionDimensions + attrs["base_values"] = ( + raw_model.treeEnsembleRegressor.treeEnsemble.basePredictionValue + ) + attrs["n_targets"] = ( + raw_model.treeEnsembleRegressor.treeEnsemble.numPredictionDimensions + ) attrs["post_transform"] = get_onnx_tree_post_transform( raw_model.treeEnsembleRegressor.postEvaluationTransform ) diff --git a/onnxmltools/convert/sparkml/ops_input_output.py b/onnxmltools/convert/sparkml/ops_input_output.py index ae241667..d74f95dd 100644 --- a/onnxmltools/convert/sparkml/ops_input_output.py +++ b/onnxmltools/convert/sparkml/ops_input_output.py @@ -134,12 +134,16 @@ def build_io_name_map(): lambda model: [model.getOrDefault("predictionCol")], ), "pyspark.ml.feature.ImputerModel": ( - lambda model: model.getOrDefault("inputCols") - if model.isSet("inputCols") - else [model.getOrDefault("inputCol")], - lambda model: model.getOrDefault("outputCols") - if model.isSet("outputCols") - else [model.getOrDefault("outputCol")], + lambda model: ( + model.getOrDefault("inputCols") + if model.isSet("inputCols") + else [model.getOrDefault("inputCol")] + ), + lambda model: ( + model.getOrDefault("outputCols") + if model.isSet("outputCols") + else [model.getOrDefault("outputCol")] + ), ), "pyspark.ml.feature.MaxAbsScalerModel": ( lambda model: [model.getOrDefault("inputCol")], @@ -177,20 +181,28 @@ def build_io_name_map(): ], ), "pyspark.ml.feature.OneHotEncoderModel": ( - lambda model: model.getOrDefault("inputCols") - if model.isSet("inputCols") - else [model.getOrDefault("inputCol")], - lambda model: model.getOrDefault("outputCols") - if model.isSet("outputCols") - else [model.getOrDefault("outputCol")], + lambda model: ( + model.getOrDefault("inputCols") + if model.isSet("inputCols") + else [model.getOrDefault("inputCol")] + ), + lambda model: ( + model.getOrDefault("outputCols") + if model.isSet("outputCols") + else [model.getOrDefault("outputCol")] + ), ), "pyspark.ml.feature.StringIndexerModel": ( - lambda model: model.getOrDefault("inputCols") - if model.isSet("inputCols") - else [model.getOrDefault("inputCol")], - lambda model: model.getOrDefault("outputCols") - if model.isSet("outputCols") - else [model.getOrDefault("outputCol")], + lambda model: ( + model.getOrDefault("inputCols") + if model.isSet("inputCols") + else [model.getOrDefault("inputCol")] + ), + lambda model: ( + model.getOrDefault("outputCols") + if model.isSet("outputCols") + else [model.getOrDefault("outputCol")] + ), ), "pyspark.ml.feature.VectorAssembler": ( lambda model: model.getOrDefault("inputCols"), diff --git a/onnxmltools/convert/xgboost/shape_calculators/Classifier.py b/onnxmltools/convert/xgboost/shape_calculators/Classifier.py index c64d9743..c245633f 100644 --- a/onnxmltools/convert/xgboost/shape_calculators/Classifier.py +++ b/onnxmltools/convert/xgboost/shape_calculators/Classifier.py @@ -27,7 +27,7 @@ def calculate_xgboost_classifier_output_shapes(operator): objective = params["objective"] n_estimators = get_n_estimators_classifier(xgb_node, params, js_trees) num_class = params.get("num_class", None) - + if objective == "binary:logistic": ncl = 2 elif num_class is not None: diff --git a/pyproject.toml b/pyproject.toml index 79f48d06..ae0a2b72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ exclude = [ # Same as Black. line-length = 95 -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 10 -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "**/__init__.py" = ["F401"] "onnxmltools/convert/coreml/operator_converters/GLMClassifier.py" = ["E501"] "onnxmltools/convert/coreml/operator_converters/SVC.py" = ["E501"] diff --git a/tests/xgboost/test_xgboost_issues.py b/tests/xgboost/test_xgboost_issues.py index fac5a8f0..47a2e08b 100644 --- a/tests/xgboost/test_xgboost_issues.py +++ b/tests/xgboost/test_xgboost_issues.py @@ -41,7 +41,9 @@ def xgbregressor_shape_calculator(operator): initial_type = [("float_input", FloatTensorType([None, X.shape[1]]))] # Convert XGBoost model to ONNX - onnx_model = convert_sklearn(model, initial_types=initial_type, target_opset=12) + onnx_model = convert_sklearn( + model, initial_types=initial_type, target_opset={"": 12, "ai.onnx.ml": 3} + ) self.assertIn("dim_value: 2", str(onnx_model.graph.output)) sess = onnxruntime.InferenceSession(