Python 3.11 support (#887)

Author: liudmylaru

.github/workflows/run-operators-unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
+name: "[Py3.8-3.11] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-py38-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] tests/unitary/**"
+name: "[Py3.8][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -44,6 +44,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore  tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"

.github/workflows/run-unittests-py39-py310.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9][Py3.10] - tests/unitary/**"
+name: "[Py3.9-3.11] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"
@@ -46,6 +46,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/operator/pii \
               --ignore tests/unitary/with_extras/hpo

.github/workflows/test-env-setup/action.yml

@@ -7,6 +7,11 @@ runs:
   using: composite
   steps:
     - shell: bash
+      env:
+        # torch > v2.1.0 brings nvidia dependency by default, we want to install torch for cpu for tests.
+        # Fot that --index-url https://download.pytorch.org/whl/cpu needed - https://pytorch.org/get-started/locally/.
+        # Setting env variable here instead of flag --index-url (docs: https://pip.pypa.io/en/stable/cli/pip_install/):
+        PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu"
       run: |
         set -x # print commands that are executed
 

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
     # ruff
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.4.9
+      rev: v0.5.0
       hooks:
           - id: ruff
             types_or: [ python, pyi, jupyter ]
@@ -13,7 +13,7 @@ repos:
             exclude: ^docs/
     # Standard hooks
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.6.0
       hooks:
           - id: check-ast
             exclude: ^docs/
@@ -42,7 +42,7 @@ repos:
             files: ^docs/
     # Hardcoded secrets and ocids detector
     - repo: https://github.com/gitleaks/gitleaks
-      rev: v8.17.0
+      rev: v8.18.4
       hooks:
           - id: gitleaks
             exclude: .github/workflows/reusable-actions/set-dummy-conf.yml

dev-requirements.txt

@@ -1,3 +1,5 @@
+# Do not add test dependencies here. Use pyproject.toml [project.optional-dependencies] 'testsuite' section.
+# Reason - it is flexible to specify different version for specific python
 -r test-requirements.txt
 -e ".[aqua,bds,data,geo,huggingface,llm,notebook,onnx,opctl,optuna,spark,tensorflow,text,torch,viz]"
 -e ".[testsuite]"

pyproject.toml

@@ -48,6 +48,7 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
 ]
 
 # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/
@@ -61,8 +62,8 @@ dependencies = [
   "fsspec>=0.8.7",
   "gitpython>=3.1.2",
   "jinja2>=2.11.2",
-  "matplotlib>=3.1.3, <=3.8.4",
-  "numpy>=1.19.2, <2.0.0",
+  "matplotlib>=3.1.3,<=3.8.4",
+  "numpy>=1.19.2,<2.0.0",
   "oci>=2.125.3",
   "ocifs>=1.1.3",
   "pandas>1.2.1; python_version<'3.9'", # starting pandas v2.1.0 requires-python = '>=3.9'
@@ -79,7 +80,7 @@ dependencies = [
 # Copied from extras_require list in setup.py, setup.py got removed in favor of this config file
 bds = ["hdfs[kerberos]", "ibis-framework[impala]", "sqlalchemy"]
 boosted = [
-  "lightgbm<4.0.0", # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm",
   "xgboost",
 ]
 data = [
@@ -89,18 +90,24 @@ data = [
   "openpyxl>=3.0.7",
   "oracledb>=1.0",
   "pandavro>=1.6.0",
-  "sqlalchemy>=1.4.1, <=1.4.46",
+  "sqlalchemy>=1.4.1,<=1.4.46",
+]
+geo = [
+  "geopandas<1.0.0",  # in v1.0.0 removed the built-in dataset 'naturalearth_lowres', fix when relax version of geopandas needed
+  "oracle_ads[viz]"
+]
+huggingface = [
+  "transformers",
+  "tf-keras"  # Keras 3 installed in py3.11+, but this is not yet supported in Transformers. Need to install the backwards-compatible tf-keras
 ]
-geo = ["geopandas", "oracle_ads[viz]"]
-huggingface = ["transformers"]
 notebook = ["ipython>=7.23.1, <8.0", "ipywidgets~=7.6.3"]
 onnx = [
-  "lightgbm<4.0.0",            # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
-  "onnx>=1.12.0",
+  "lightgbm",
+  "onnx>=1.12.0,<=1.15.0",  # v 1.15.0 set base on onnxrutime version and onnx opset support - https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support
   "onnxmltools>=1.10.0",
-  "onnxruntime>=1.10.0,<1.16", # v1.16 introduced issues https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime~=1.17.0,!=1.16.0",  # v1.17.0 used in Oracle Database 23ai; avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
   "oracle_ads[viz]",
-  "protobuf<=3.20",
+  "protobuf",
   "skl2onnx>=1.10.4",
   "tf2onnx",
   "xgboost<=1.7",
@@ -119,11 +126,21 @@ opctl = [
 ]
 optuna = ["optuna==2.9.0", "oracle_ads[viz]"]
 spark = ["pyspark>=3.0.0"]
-tensorflow = ["oracle_ads[viz]", "tensorflow"]
-text = ["spacy", "wordcloud>=1.8.1"]
-torch = ["oracle_ads[viz]", "torch", "torchvision"]
+tensorflow = [
+  "oracle_ads[viz]",
+  "tensorflow<=2.15.1"  # v2.16.1 with consequence on tf2onnx v1.16.1 (latest) has an issue with Keras 3 installed in py3.11+ (https://github.com/onnx/tensorflow-onnx/issues/2319)
+]
+text = [
+  "spacy>=3.4.2",   # the first version of spacy that supports python 3.11 is spacy v3.4.2
+  "wordcloud>=1.8.1"
+]
+torch = [
+  "oracle_ads[viz]",
+  "torch",
+  "torchvision"
+]
 viz = [
-  "bokeh>=3.0.0, <3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
+  "bokeh>=3.0.0,<3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
   "folium>=0.12.1",
   "graphviz<0.17",
   "scipy>=1.5.4",
@@ -182,13 +199,9 @@ aqua = ["jupyter_server"]
 # trying to use. Ref - https://pip.pypa.io/en/stable/topics/dependency-resolution/#possible-ways-to-reduce-backtracking.
 # Revisit this section continuously and update to recent version of libraries. focus on pyt3.9/3.10 versions.
 testsuite = [
-  "dask==2023.5.0; python_version=='3.8'",
-  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
   "arff",
   "category_encoders==2.6.3",  # set version to avoid backtracking
   "cohere==4.53",  # set version to avoid backtracking
-  "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
-  "dask==2023.5.0; python_version=='3.8'",
   "faiss-cpu",
   "fastparquet==2024.2.0",  # set version to avoid backtracking
   "imbalanced-learn",
@@ -199,10 +212,11 @@ testsuite = [
   "opensearch-py",
   "pdfplumber",
   "py4j",
-  "pyarrow",
+  "pyarrow>=15.0.0",
   "statsmodels; python_version=='3.8'",
   "statsmodels>=0.14.1; python_version>='3.9'",  # cython3.0 compatibility added in v0.14.1
   "tables",
+  "tables>3.9.0; python_version>='3.9'",
   "xlrd>=1.2.0",
 ]
 

test-requirements.txt

@@ -1,3 +1,5 @@
+# Include here only libraries required to run test methods, like mock, pytest, coverage etc.
+# Add dependencies for ADS to pyproject.toml [project.optional-dependencies] 'testsuite' section.
 -e .
 click
 coverage
@@ -8,5 +10,4 @@ pip
 pytest
 pytest-cov
 pytest-xdist
-kubernetes
 ruff

tests/unitary/default_setup/common/test_common_ADSData.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """
 Contains tests for ads.common.data
 """
+
 import os
 import pandas as pd
 import pytest
 import unittest
 
 from ads.common.data import ADSData
 
+
 #
 # run with:
 #  python -m pytest -v -p no:warnings --cov-config=.coveragerc --cov=./ --cov-report html /home/datascience/advanced-ds/tests/unitary/test_common_data_ADSData.py
@@ -112,7 +114,7 @@ def test_ADSData_build_bad_input(self):
         Test corner cases and error handling
         """
         bad_input = [(None, None), ("test", None), (None, "test")]
-        for (X, y) in bad_input:
+        for X, y in bad_input:
             with pytest.raises(ValueError):
                 ADSData.build(X, y)
 
@@ -149,22 +151,6 @@ def test_ADSData_build_valid_input_using_target_vector(self):
         assert expected.y.name == "target"
         assert expected.y.shape == (3,)
 
-    @pytest.mark.skipif("NoDependency" in os.environ, reason="skip for dependency test")
-    def test_ADSData_build_valid_input_dask_dataframe(self):
-        """
-        Ensures build method takes dask dataframe
-        """
-        import dask
-
-        X = dask.datasets.timeseries().drop("y", axis=1)
-        y = dask.datasets.timeseries()["y"]
-        expected = ADSData.build(X, y)
-        assert sorted(expected.X.columns.tolist()) == sorted(["id", "name", "x"])
-        assert expected.X.shape[0] == 2592000
-        assert expected.X.shape[1] == 3
-        assert expected.y.name == "y"
-        assert expected.y.shape[0] == 2592000
-
     @pytest.mark.skip("api change. this test should be re-written.")
     def test_ADSData_build_with_data(self):
         """

tests/unitary/default_setup/feature_types/test_feature_domain_schema.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -524,24 +524,6 @@ def test_populate_schema_list(self):
                 self.model_artifact.populate_schema(X_sample=list(self.X))
                 mock_warning.assert_called()
 
-    @pytest.mark.skipif("NoDependency" in os.environ, reason="skip for dependency test")
-    def test_populate_schema_dask_tuple(self):
-        import dask.dataframe as dd
-
-        self.model_artifact.populate_schema(
-            X_sample=dd.from_array(self.X), y_sample=tuple(self.y)
-        )
-        assert isinstance(self.model_artifact.schema_input, Schema)
-        assert isinstance(self.model_artifact.schema_output, Schema)
-
-        # Test wide data
-        with patch.object(Schema, "validate_size", side_effect=SchemaSizeTooLarge(100)):
-            with patch.object(logger, "warning") as mock_warning:
-                self.model_artifact.populate_schema(
-                    X_sample=dd.from_array(self.X), y_sample=tuple(self.y)
-                )
-                mock_warning.assert_called()
-
     def test_simple_constraint(self):
         self.df["sepal length (cm)"].ads.feature_type = ["category"]
         domain = self.df["sepal length (cm)"].ads.feature_domain()