Master ci py312 new

.githooks/pre-push

@@ -12,5 +12,5 @@ start_time=`date +%s`
 tox -e sphinx,doc8 --parallel all
 ./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
 start_time=`date +%s`
-tox -e py38,py39,py310 --parallel all -- tests/unit
-./ci-scripts/displaytime.sh 'py38,py39,py310 unit' $start_time
+tox -e py39,py310,py311,py312 --parallel all -- tests/unit
+./ci-scripts/displaytime.sh 'py39,py310,py311,py312 unit' $start_time

.github/workflows/codebuild-ci-health.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          python-version: ["py38", "py39", "py310", "py311"]
+          python-version: ["py39", "py310", "py311","py312"]
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/codebuild-ci.yml

@@ -63,7 +63,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          python-version: ["py38","py39","py310","py311"]
+          python-version: ["py39","py310","py311","py312"]
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4

doc/requirements.txt

@@ -1,8 +1,8 @@
 sphinx==5.1.1
 sphinx-rtd-theme==0.5.0
 docutils==0.15.2
-packaging==20.9
-jinja2==3.1.6
+packaging>=23.0,<25
+jinja2==3.1.4
 schema==0.7.5
 accelerate>=0.24.1,<=0.27.0
 graphene<4.0

pyproject.toml

@@ -7,7 +7,7 @@ name = "sagemaker"
 dynamic = ["version", "optional-dependencies"]
 description = "Open source library for training and deploying models on Amazon SageMaker."
 readme = "README.rst"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 authors = [
   { name = "Amazon Web Services" },
 ]
@@ -25,10 +25,10 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Natural Language :: English",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
   "attrs>=23.1.0,<24",
@@ -39,15 +39,15 @@ dependencies = [
   "google-pasta",
   "importlib-metadata>=1.4.0,<7.0",
   "jsonschema",
-  "numpy>=1.9.0,<2.0",
+  "numpy>=1.26.0,<2.0",
   "omegaconf>=2.2,<=2.3",
-  "packaging>=20.0",
+  "packaging>=23.0,<25",
   "pandas",
   "pathos",
   "platformdirs",
   "protobuf>=3.12,<6.0",
   "psutil",
-  "PyYAML~=6.0",
+  "PyYAML>=6.0.1",
   "requests",
   "sagemaker-core>=1.0.17,<2.0.0",
   "schema",

requirements/extras/local_requirements.txt

@@ -1,3 +1,3 @@
 urllib3>=1.26.8,<3.0.0
 docker>=5.0.2,<8.0.0
-PyYAML>=5.4.1,<7
+PyYAML>=6.0.1,<7

requirements/extras/scipy_requirements.txt

@@ -1 +1 @@
-scipy==1.10.1
+scipy==1.11.3

requirements/extras/test_requirements.txt

@@ -1,5 +1,5 @@
 tox==3.24.5
-numpy>=1.24.0
+numpy>=1.26.0,<2.0
 build[virtualenv]==1.2.1
 flake8==4.0.1
 pytest==6.2.5
@@ -14,7 +14,7 @@ awslogs==0.14.0
 black==24.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.9.3
+apache-airflow==2.10.4
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0
@@ -26,14 +26,14 @@ pandas==1.4.4
 scikit-learn==1.3.0
 cloudpickle==2.2.1
 jsonpickle<4.0.0
-PyYAML==6.0
+PyYAML>=6.0.1
 # TODO find workaround
 xgboost>=1.6.2,<=1.7.6
 pillow>=10.0.1,<=11
 opentelemetry-proto==1.27.0
 protobuf==4.25.5
-tensorboard>=2.9.0,<=2.15.2
-transformers==4.48.0
+tensorboard>=2.16.2,<=2.18.0
+transformers==4.46.1
 sentencepiece==0.1.99
 # https://github.com/triton-inference-server/server/issues/6246
 tritonclient[http]<2.37.0
@@ -42,7 +42,7 @@ onnx==1.17.0
 nbformat>=5.9,<6
 accelerate>=0.24.1,<=0.27.0
 schema==0.7.5
-tensorflow>=2.9.0,<=2.15.1
+tensorflow>=2.16.2,<=2.18.0
 mlflow>=2.12.2,<2.13
 huggingface_hub==0.26.2
 uvicorn>=0.30.1

src/sagemaker/serve/utils/conda_in_process.yml

@@ -12,15 +12,15 @@ dependencies:
     - boto3>=1.34.142,<2.0
     - cloudpickle==2.2.1
     - google-pasta
-    - numpy>=1.9.0,<2.0
+    - numpy>=1.26.0,<2.0
     - protobuf>=3.12,<5.0
     - smdebug_rulesconfig==1.0.1
     - importlib-metadata>=1.4.0,<7.0
-    - packaging>=20.0
+    - packaging>=23.0,<25
     - pandas
     - pathos
     - schema
-    - PyYAML~=6.0
+    - PyYAML>=6.0.1
     - jsonschema
     - platformdirs
     - tblib>=1.7.0,<4
@@ -43,7 +43,7 @@ dependencies:
             - colorama>=0.4.4
             - contextlib2>=21.6.0
             - decorator>=5.1.1
-            - dill>=0.3.6
+            - dill>=0.3.9
             - docutils>=0.16
             - entrypoints>=0.4
             - filelock>=3.11.0
@@ -82,7 +82,7 @@ dependencies:
             - python-dateutil>=2.8.2
             - pytz>=2023.3
             - pytz-deprecation-shim>=0.1.0.post0
-            - pyyaml>=5.4.1
+            - pyyaml>=6.0.1
             - regex>=2023.3.23
             - requests>=2.28.2
             - rich>=13.3.4

src/sagemaker/serve/utils/in_process_requirements.txt

@@ -11,7 +11,7 @@ cloudpickle==2.2.1
 colorama>=0.4.4
 contextlib2>=21.6.0
 decorator>=5.1.1
-dill>=0.3.6
+dill>=0.3.9
 docutils>=0.16
 entrypoints>=0.4
 filelock>=3.11.0
@@ -50,7 +50,7 @@ pyrsistent>=0.19.3
 python-dateutil>=2.8.2
 pytz>=2023.3
 pytz-deprecation-shim>=0.1.0.post0
-pyyaml>=5.4.1
+pyyaml>=6.0.1
 regex>=2023.3.23
 requests>=2.28.2
 rich>=13.3.4

tests/data/pipeline/model_step/pytorch_mnist/requirements.txt

@@ -1 +1 @@
-scipy>=1.8.1
+scipy>=1.11.3

tests/data/remote_function/requirements.txt

@@ -1 +1 @@
-scipy==1.10.1
+scipy==1.11.3

tests/data/serve_resources/mlflow/pytorch/conda.yaml

@@ -9,7 +9,7 @@ dependencies:
   - cffi==1.16.0
   - cloudpickle==2.2.1
   - defusedxml==0.7.1
-  - dill==0.3.8
+  - dill==0.3.9
   - gmpy2==2.1.2
   - numpy==1.26.4
   - opt-einsum==3.3.0

tests/data/serve_resources/mlflow/pytorch/requirements.txt

@@ -3,11 +3,11 @@ astunparse==1.6.3
 cffi==1.16.0
 cloudpickle==2.2.1
 defusedxml==0.7.1
-dill==0.3.8
+dill==0.3.9
 gmpy2==2.1.2
-numpy==1.24.4
+numpy==1.26.4
 opt-einsum==3.3.0
-packaging==21.3
+packaging>=23.0,<25
 pandas==2.2.1
 pyyaml==6.0.1
 requests==2.32.2

tests/data/serve_resources/mlflow/xgboost/requirements.txt

@@ -1,8 +1,8 @@
 mlflow==2.13.2
 lz4==4.3.2
-numpy==1.24.4
+numpy==1.26.4
 pandas==2.0.3
 psutil==5.9.8
 scikit-learn==1.3.2
-scipy==1.10.1
+scipy==1.11.3
 xgboost==1.7.1

tests/data/workflow/requirements.txt

@@ -1 +1 @@
-scipy==1.10.1
+scipy==1.11.3

tests/integ/sagemaker/experiments/test_run.py

@@ -32,7 +32,7 @@
 from sagemaker.experiments.trial_component import _TrialComponent
 from sagemaker.sklearn import SKLearn
 from sagemaker.utils import retry_with_backoff, unique_name_from_base
-from tests.integ.sagemaker.experiments.helpers import name, cleanup_exp_resources
+from tests.integ.sagemaker.experiments.helpers import name, cleanup_exp_resources, clear_run_context
 from sagemaker.experiments.run import (
     RUN_NAME_BASE,
     DELIMITER,
@@ -55,7 +55,7 @@ def artifact_file_path(tempdir):
 metric_name = "Test-Local-Init-Log-Metric"
 
 
-def test_local_run_with_load(sagemaker_session, artifact_file_path):
+def test_local_run_with_load(sagemaker_session, artifact_file_path, clear_run_context):
     exp_name = f"My-Local-Exp-{name()}"
     with cleanup_exp_resources(exp_names=[exp_name], sagemaker_session=sagemaker_session):
         # Run name is not provided, will create a new TC
@@ -86,7 +86,9 @@ def verify_load_run():
         retry_with_backoff(verify_load_run, 4)
 
 
-def test_two_local_run_init_with_same_run_name_and_different_exp_names(sagemaker_session):
+def test_two_local_run_init_with_same_run_name_and_different_exp_names(
+    sagemaker_session, clear_run_context
+):
     exp_name1 = f"my-two-local-exp1-{name()}"
     exp_name2 = f"my-two-local-exp2-{name()}"
     run_name = "test-run"
@@ -124,7 +126,9 @@ def test_two_local_run_init_with_same_run_name_and_different_exp_names(sagemaker
         ("my-test4", "test-run", "run-display-name-test"),  # with supplied display name
     ],
 )
-def test_run_name_vs_trial_component_name_edge_cases(sagemaker_session, input_names):
+def test_run_name_vs_trial_component_name_edge_cases(
+    sagemaker_session, input_names, clear_run_context
+):
     exp_name, run_name, run_display_name = input_names
     with cleanup_exp_resources(exp_names=[exp_name], sagemaker_session=sagemaker_session):
         with Run(
@@ -177,6 +181,7 @@ def test_run_from_local_and_train_job_and_all_exp_cfg_match(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. The 1st Run created locally and its exp config was auto passed to the job
@@ -277,6 +282,7 @@ def test_run_from_local_and_train_job_and_exp_cfg_not_match(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. The 1st Run created locally and its exp config was auto passed to the job
@@ -363,6 +369,7 @@ def test_run_from_train_job_only(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. No Run created locally or specified in experiment config
@@ -413,6 +420,7 @@ def test_run_from_processing_job_and_override_default_exp_config(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. The 1st Run (run) created locally
@@ -492,6 +500,7 @@ def test_run_from_transform_job(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. The 1st Run (run) created locally
@@ -573,6 +582,7 @@ def test_load_run_auto_pass_in_exp_config_to_job(
     execution_role,
     sagemaker_client_config,
     sagemaker_metrics_config,
+    clear_run_context,
 ):
     # Notes:
     # 1. In local side, load the Run created previously and invoke a job under the load context
@@ -621,7 +631,7 @@ def test_load_run_auto_pass_in_exp_config_to_job(
         )
 
 
-def test_list(run_obj, sagemaker_session):
+def test_list(run_obj, sagemaker_session, clear_run_context):
     tc1 = _TrialComponent.create(
         trial_component_name=f"non-run-tc1-{name()}",
         sagemaker_session=sagemaker_session,
@@ -643,7 +653,7 @@ def test_list(run_obj, sagemaker_session):
     assert run_tcs[0].experiment_config == run_obj.experiment_config
 
 
-def test_list_twice(run_obj, sagemaker_session):
+def test_list_twice(run_obj, sagemaker_session, clear_run_context):
     tc1 = _TrialComponent.create(
         trial_component_name=f"non-run-tc1-{name()}",
         sagemaker_session=sagemaker_session,
@@ -720,8 +730,8 @@ def _generate_processor(
     )
     return FrameworkProcessor(
         estimator_cls=PyTorch,
-        framework_version="1.10",
-        py_version="py38",
+        framework_version="1.13.1",
+        py_version="py39",
         instance_count=1,
         instance_type="ml.m5.xlarge",
         role=execution_role,

tests/integ/sagemaker/workflow/test_workflow.py

@@ -1122,8 +1122,8 @@ def test_model_registration_with_tuning_model(
         entry_point=entry_point,
         source_dir=base_dir,
         role=role,
-        framework_version="1.10",
-        py_version="py38",
+        framework_version="1.13.1",
+        py_version="py39",
         instance_count=instance_count,
         instance_type=instance_type,
         sagemaker_session=pipeline_session,
@@ -1159,8 +1159,8 @@ def test_model_registration_with_tuning_model(
         ),
         entry_point=entry_point,
         source_dir=base_dir,
-        framework_version="1.10",
-        py_version="py38",
+        framework_version="1.13.1",
+        py_version="py39",
         sagemaker_session=pipeline_session,
     )
     step_model_regis_args = model.register(

tests/unit/sagemaker/huggingface/test_llm_utils.py

@@ -65,7 +65,7 @@ def test_huggingface_model_metadata_unauthorized_exception(self, mock_urllib):
             "Trying to access a gated/private HuggingFace model without valid credentials. "
             "Please provide a HUGGING_FACE_HUB_TOKEN in env_vars"
         )
-        self.assertEquals(expected_error_msg, str(context.exception))
+        self.assertEqual(expected_error_msg, str(context.exception))
 
     @patch("sagemaker.huggingface.llm_utils.urllib")
     def test_huggingface_model_metadata_general_exception(self, mock_urllib):
@@ -76,7 +76,7 @@ def test_huggingface_model_metadata_general_exception(self, mock_urllib):
         expected_error_msg = (
             f"Did not find model metadata for the following HuggingFace Model ID {MOCK_HF_ID}"
         )
-        self.assertEquals(expected_error_msg, str(context.exception))
+        self.assertEqual(expected_error_msg, str(context.exception))
 
     @patch("huggingface_hub.snapshot_download")
     def test_download_huggingface_model_metadata(self, mock_snapshot_download):

tests/unit/sagemaker/jumpstart/constants.py

@@ -14360,7 +14360,7 @@
             "jmespath==1.0.1",
             "jsonschema==4.17.3",
             "multiprocess==0.70.14",
-            "numpy==1.24.3",
+            "numpy==1.26.4",
             "oscrypto==1.3.0",
             "packaging==23.1",
             "pandas==2.0.2",