Support Python versions up to 3.12

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
+name: "[Py3.8-3.12] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4
@@ -60,5 +60,8 @@ jobs:
           source /home/runner/.bashrc
           conda install python=${{ matrix.python-version }}
           pip install -r test-requirements.txt
+          if [[ ${{ matrix.python-version }} == "3.12" ]]; then
+            pip install setuptools
+          fi
           conda list
           python -m pytest -v -p no:warnings --durations=5 tests/unitary/default_setup

.github/workflows/run-unittests-py38-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] tests/unitary/**"
+name: "[Py3.8][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:

.github/workflows/run-unittests-py39-py310.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9][Py3.10] - tests/unitary/**"
+name: "[Py3.9-3.12] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        # dask, which is used for tests and some use-cases incompatible with latest py3.11.9, py3.12.3 (https://github.com/dask/dask/issues/11038)
+        # To fix issues with dask, set python to "3.11.8", "3.12.2". Relax it to "3.11", "3.12" after issue with dask resolved.
+        # To relax python to "3.11", "3.12" may need to relax dask. But oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1".
+        python-version: ["3.9", "3.10",  "3.11.8", "3.12.2"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"

ads/common/model.py

@@ -4,7 +4,6 @@
 # Copyright (c) 2020, 2022 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from distutils import dir_util
 import os
 import shutil
 from collections.abc import Iterable
@@ -495,7 +494,8 @@ def prepare(
                 ),
                 os.path.join(target_dir, ".model-ignore"),
             )
-            dir_util._path_created = {}
+            # from distutils import dir_util
+            # dir_util._path_created = {}
 
             progress.update("Serializing model")
             # Transform the data to be onnx-ready

ads/common/model_artifact.py

@@ -38,7 +38,6 @@
 import oci.data_science
 import oci.exceptions
 import pandas as pd
-import pkg_resources
 import yaml
 
 from ads.common.decorator.runtime_dependency import (
@@ -982,6 +981,7 @@ def install_requirements(self, conflict_strategy=ConflictStrategy.IGNORE):
             IGNORE: Use the installed version in  case of a conflict.
             UPDATE: Force update dependency to the version required by model artifact in case of conflict.
         """
+        import pkg_resources
         importlib.reload(pkg_resources)
         from pkg_resources import DistributionNotFound, VersionConflict
 

ads/common/model_export_util.py

@@ -39,7 +39,6 @@
 from ads.common.model_artifact import ModelArtifact
 from ads.model.model_metadata import UseCaseType
 from ads.feature_engineering.schema import DataSizeTooWide
-from pkg_resources import DistributionNotFound, get_distribution
 
 from ads.model.transformer.onnx_transformer import (
     ONNXTransformer,
@@ -206,6 +205,8 @@ def prepare_generic_model(
     model_artifact: ads.model_artifact.model_artifact
         A generic model artifact
     """
+    from pkg_resources import DistributionNotFound, get_distribution
+
     if "function_artifacts" in kwargs:
         if (
             fn_artifact_files_included

ads/common/utils.py

@@ -22,6 +22,7 @@
 import tempfile
 from datetime import datetime
 from enum import Enum
+from importlib.metadata import version
 from io import DEFAULT_BUFFER_SIZE
 from pathlib import Path
 from textwrap import fill
@@ -55,7 +56,7 @@
 
 # For Model / Model Artifact libraries
 lib_translator = {"sklearn": "scikit-learn"}
-module_ignore = ["builtins", "ads", "automl", "mlx"]
+module_ignore = ["builtins", "ads", "automl", "mlx", "automlx"]
 
 # up-sample if length of dataframe is less than or equal to MAX_LEN_FOR_UP_SAMPLING
 MAX_LEN_FOR_UP_SAMPLING = 5000
@@ -905,8 +906,6 @@ def extract_lib_dependencies_from_model(model) -> dict:
     -------
     Dict: A dictionary of library dependencies.
     """
-    from pkg_resources import get_distribution
-
     module_versions = {}
     modules_to_include = set(
         mod.__module__.split(".")[0]
@@ -917,7 +916,7 @@ def extract_lib_dependencies_from_model(model) -> dict:
         if mod not in module_ignore:
             try:
                 mod_name = lib_translator.get(mod, mod)
-                module_versions[mod_name] = get_distribution(mod_name).version
+                module_versions[mod_name] = version(mod_name)
             except:
                 pass
     return module_versions

ads/model/serde/model_serializer.py

@@ -465,6 +465,7 @@ def skl2onnx_convert_lightgbm(scope, operator, container):
                         **kwargs,
                     )
                 except Exception as e:
+                    logger.exception("Exception details:")
                     raise ValueError(
                         "`initial_types` can not be autodetected. Please directly pass `initial_types`."
                     )
@@ -792,6 +793,7 @@ def _to_onnx(
                         **kwargs,
                     )
                 except:
+                    logger.exception("Exception details:")
                     raise ValueError(
                         "`initial_types` can not be autodetected. Please directly pass `initial_types`."
                     )
@@ -815,6 +817,7 @@ def _to_onnx(
                         **kwargs,
                     )
                 except:
+                    logger.exception("Exception details:")
                     raise ValueError(
                         "`initial_types` can not be autodetected. Please directly pass `initial_types`."
                     )

ads/opctl/backend/ads_ml_job.py

@@ -11,7 +11,6 @@
 import shutil
 import tempfile
 import time
-from distutils import dir_util
 from typing import Dict, Tuple, Union
 
 from ads.common.auth import AuthContext, AuthType, create_signer
@@ -291,7 +290,7 @@ def _run_with_conda_pack(self, payload: Job, src_folder: str) -> Tuple[str, str]
         if ConfigResolver(self.config)._is_ads_operator():
             with tempfile.TemporaryDirectory() as td:
                 os.makedirs(os.path.join(td, "operators"), exist_ok=True)
-                dir_util.copy_tree(
+                shutil.copytree(
                     src_folder,
                     os.path.join(td, "operators", os.path.basename(src_folder)),
                 )
@@ -311,7 +310,7 @@ def _run_with_conda_pack(self, payload: Job, src_folder: str) -> Tuple[str, str]
                 run_id = job.run().id
         else:
             with tempfile.TemporaryDirectory() as td:
-                dir_util.copy_tree(
+                shutil.copytree(
                     src_folder, os.path.join(td, os.path.basename(src_folder))
                 )
                 payload.runtime.with_source(

pyproject.toml

@@ -48,6 +48,8 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 
 # PEP 508 – Dependency specification for Python Software Packages - https://peps.python.org/pep-0508/
@@ -79,7 +81,7 @@ dependencies = [
 # Copied from extras_require list in setup.py, setup.py got removed in favor of this config file
 bds = ["hdfs[kerberos]", "ibis-framework[impala]", "sqlalchemy"]
 boosted = [
-  "lightgbm<4.0.0", # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm",
   "xgboost",
 ]
 data = [
@@ -92,15 +94,18 @@ data = [
   "sqlalchemy>=1.4.1, <=1.4.46",
 ]
 geo = ["geopandas", "oracle_ads[viz]"]
-huggingface = ["transformers"]
+huggingface = [
+  "transformers",
+  "tf-keras"  # Keras 3 installed in py3.11+, but this is not yet supported in Transformers. Need to install the backwards-compatible tf-keras
+]
 notebook = ["ipython>=7.23.1, <8.0", "ipywidgets~=7.6.3"]
 onnx = [
-  "lightgbm<4.0.0",            # relax when the official releases of skl2onnx (v1.16.0) and onnxmltools (1.11.3), https://github.com/sdpython/mlprodict/issues/488
+  "lightgbm",
   "onnx>=1.12.0",
   "onnxmltools>=1.10.0",
-  "onnxruntime>=1.10.0,<1.16", # v1.16 introduced issues https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
+  "onnxruntime>=1.10.0,!=1.16.0", # avoid v1.16 https://github.com/microsoft/onnxruntime/issues/17631, revealed by unit tests
   "oracle_ads[viz]",
-  "protobuf<=3.20",
+  "protobuf",
   "skl2onnx>=1.10.4",
   "tf2onnx",
   "xgboost<=1.7",
@@ -120,7 +125,10 @@ opctl = [
 optuna = ["optuna==2.9.0", "oracle_ads[viz]"]
 spark = ["pyspark>=3.0.0"]
 tensorflow = ["oracle_ads[viz]", "tensorflow"]
-text = ["spacy", "wordcloud>=1.8.1"]
+text = [
+  "spacy>=3.4.2",   # the first version of spacy that supports python 3.11 is spacy v3.4.2
+  "wordcloud>=1.8.1"
+]
 torch = ["oracle_ads[viz]", "torch", "torchvision"]
 viz = [
   "bokeh>=3.0.0, <3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
@@ -190,19 +198,20 @@ testsuite = [
   "dask==2023.10.1; python_version>='3.9'", # oci-cli depends on click==8.0.4, dask>2023.10.1 depends on "click>=8.1"
   "dask==2023.5.0; python_version=='3.8'",
   "faiss-cpu",
-  "fastparquet==2024.2.0",  # set version to avoid backtracking
+  "fastparquet==2024.2.0",  # set version to avoid backtracking, change it to fresher version from time to time
   "imbalanced-learn",
   "lxml",
   "mysql-connector-python",
   "nltk",
   "notebook==6.4.12",
   "opensearch-py",
+  "setuptools; python_version>='3.12'",
   "pdfplumber",
   "py4j",
-  "pyarrow",
+  "pyarrow>=15.0.0",
   "statsmodels; python_version=='3.8'",
   "statsmodels>=0.14.1; python_version>='3.9'",  # cython3.0 compatibility added in v0.14.1
-  "tables",
+  "tables>3.9.0",
   "xlrd>=1.2.0",
 ]
 

tests/unitary/default_setup/model/test_onnx_transformer.py

@@ -192,4 +192,4 @@ def test__handle_missing_value_series(df):
 
 
 def teardown_module():
-    shutil.rmtree(tmp_model_dir)
+    shutil.rmtree(tmp_model_dir, ignore_errors=True)

tests/unitary/default_setup/pipeline/test_pipeline_run.py

@@ -699,7 +699,7 @@ def test_watch_service_log(self, mock_logs, mock_stream_log):
         mock_logs.return_value = ConsolidatedLog(OCILog(log_type="SERVICE"))
 
         pipeline_run.watch(log_type="service_log")
-        mock_logs.called_with(log_type="service_log")
+        mock_logs.assert_called_with(log_type="service_log")
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, "service_log")
 
     @patch.object(PipelineRun, "_PipelineRun__stream_log")
@@ -710,7 +710,7 @@ def test_watch_custom_log(self, mock_logs, mock_stream_log):
         mock_logs.return_value = ConsolidatedLog(OCILog(log_type="CUSTOM"))
 
         pipeline_run.watch(log_type="custom_log")
-        mock_logs.called_with(log_type="custom_log")
+        mock_logs.assert_called_with(log_type="custom_log")
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, "custom_log")
 
     @patch.object(PipelineRun, "_PipelineRun__stream_log")
@@ -724,7 +724,7 @@ def test_watch_both_log(self, mock_logs, mock_stream_log):
         )
 
         pipeline_run.watch()
-        mock_logs.called_with(log_type=None)
+        mock_logs.assert_called_with(log_type=None)
         mock_stream_log.assert_called_with(mock_logs.return_value, [], 3, None)
 
     def test_build_filter_expression(self):