Add support for Python=3.12 (#3501)

Co-authored-by: Bouwe Andela <[email protected]>
ESMValGroup · Oct 9, 2024 · 1d3dbd4 · 1d3dbd4
1 parent 1de5bf6
commit 1d3dbd4
Show file tree

Hide file tree

Showing 12 changed files with 51 additions and 46 deletions.
diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -27,7 +27,7 @@ jobs:
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.11"
+          python-version: "3.12"
           miniforge-version: "latest"
           miniforge-variant: Mambaforge
           use-mamba: true

diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: conda-incubator/setup-miniconda@v3

diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:

diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml
@@ -19,7 +19,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:

diff --git a/.github/workflows/pypi-build-and-deploy.yml b/.github/workflows/pypi-build-and-deploy.yml
@@ -17,10 +17,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install pep517
         run: >-
           python -m

diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
@@ -67,7 +67,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.github/workflows/test-development.yml b/.github/workflows/test-development.yml
@@ -26,7 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
@@ -45,6 +45,8 @@ jobs:
         run: conda list
       - name: Install ESMValTool
         run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Examine conda environment
+        run: conda list
       - name: Install Julia dependencies
         run: esmvaltool install Julia
       - name: Export Python minor version
@@ -72,7 +74,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/environment.yml b/environment.yml
@@ -23,12 +23,13 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
+  - esmpy  # <8.6 safe https://github.com/SciTools/iris-esmf-regrid/issues/415
   - esmvalcore 2.11.*
   - fiona
   - fire
   - fsspec
-  - gdal
+  - gdal >=3.9.0
+  - importlib_metadata <8  # https://github.com/ESMValGroup/ESMValTool/issues/3699 only for Python 3.10/11 and esmpy<8.6
   - iris >=3.6.1
   - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
@@ -40,39 +41,39 @@ dependencies:
   - nc-time-axis
   - netCDF4
   - numba
-  - numpy !=1.24.3  # severe masking bug
+  - numpy !=1.24.3,<2.0  # severe masking bug
   - openpyxl
   - packaging
-  - pandas !=2.2.0,!=2.2.1,!=2.2.2 # github.com/ESMValGroup/ESMValCore/pull/2305
+  - pandas==2.1.4  # unpin when ESMValCore released with https://github.com/ESMValGroup/ESMValCore/pull/2529
   - pip !=21.3
   - progressbar2
   - prov
-  - psyplot
-  - psy-maps
-  - psy-reg
-  - psy-simple
+  - psyplot >=1.5.0
+  - psy-maps >=1.5.0
+  - psy-reg >=1.5.0
+  - psy-simple >=1.5.0
   - pyproj >=2.1
   - pys2index  # only from conda-forge
-  - python >=3.10
+  - python >=3.10,<3.13
   - python-cdo
   - python-dateutil
   - pyyaml
-  - rasterio
+  - rasterio >=1.3.10
   - requests
   - ruamel.yaml
   - scikit-image
   - scikit-learn >= 1.4.0  # github.com/ESMValGroup/ESMValTool/issues/3504
   - scipy
   - seaborn
   - seawater
-  - shapely >=2
+  - shapely >=2.0.2
   - xarray >=0.12.0
   - xesmf >=0.7.1
   - xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779
   - xlsxwriter
   - zarr
   # Python packages needed for unit testing
-  - flake8 ==5.0.4
+  - flake8 >=6
   - pytest >=3.9,!=6.0.0rc1,!=6.0.0
   - pytest-cov
   - pytest-env
@@ -91,14 +92,14 @@ dependencies:
   - imagehash
   - isort ==5.13.2
   - pre-commit
-  - prospector
+  - prospector >=1.12  # earliest support for Python 3.12
   - pyroma
   # - vprof not on conda-forge
   - yamllint ==1.35.1
   - yapf ==0.32.0
 
   # NCL and dependencies
-  - ncl
+  - ncl >=6.6.2
   - cdo
   - imagemagick
   - nco

diff --git a/environment_osx.yml b/environment_osx.yml
@@ -22,12 +22,13 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
+  - esmpy  # <8.6 safe https://github.com/SciTools/iris-esmf-regrid/issues/415
   - esmvalcore 2.11.*
   - fiona
   - fire
   - fsspec
-  - gdal
+  - gdal >=3.9.0
+  - importlib_metadata <8  # https://github.com/ESMValGroup/ESMValTool/issues/3699 only for Python 3.10/11 and esmpy<8.6
   - iris >=3.6.1
   - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
@@ -39,39 +40,39 @@ dependencies:
   - nc-time-axis
   - netCDF4
   - numba
-  - numpy !=1.24.3  # severe masking bug
+  - numpy !=1.24.3,<2.0  # severe masking bug
   - openpyxl
   - packaging
-  - pandas !=2.2.0,!=2.2.1,!=2.2.2 # github.com/ESMValGroup/ESMValCore/pull/2305
+  - pandas==2.1.4  # unpin when ESMValCore released with https://github.com/ESMValGroup/ESMValCore/pull/2529
   - pip !=21.3
   - progressbar2
   - prov
-  - psyplot
-  - psy-maps
-  - psy-reg
-  - psy-simple
+  - psyplot >=1.5.0
+  - psy-maps >=1.5.0
+  - psy-reg >=1.5.0
+  - psy-simple >=1.5.0
   - pyproj >=2.1
   - pys2index  # only from conda-forge
-  - python >=3.10
+  - python >=3.10,<3.13
   - python-cdo
   - python-dateutil
   - pyyaml
-  - rasterio
+  - rasterio >=1.3.10
   - requests
   - ruamel.yaml
   - scikit-image
   - scikit-learn >= 1.4.0  # github.com/ESMValGroup/ESMValTool/issues/3504
   - scipy
   - seaborn
   - seawater
-  - shapely >=2
+  - shapely >=2.0.2
   - xarray >=0.12.0
   - xesmf >=0.7.1
   - xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779
   - xlsxwriter
   - zarr
   # Python packages needed for unit testing
-  - flake8 ==5.0.4
+  - flake8 >=6
   - pytest >=3.9,!=6.0.0rc1,!=6.0.0
   - pytest-cov
   - pytest-env
@@ -90,7 +91,7 @@ dependencies:
   - imagehash
   - isort ==5.13.2
   - pre-commit
-  - prospector
+  - prospector >=1.12  # earliest support for Python 3.12
   - pyroma
   # - vprof not on conda-forge
   - yamllint ==1.35.1

diff --git a/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py b/esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py
@@ -365,7 +365,7 @@ def _create_regression_plot(tas_cube,
     y_reg = reg.slope * x_reg + reg.intercept
 
     # Plot data
-    title = (f'{FEEDBACK_PARAMETERS.get(var,var)} TOA radiance for '
+    title = (f'{FEEDBACK_PARAMETERS.get(var, var)} TOA radiance for '
              f'{dataset_name}')
     filename = f'{var}_regression_{dataset_name}'
     if description is not None:

diff --git a/setup.py b/setup.py
@@ -51,17 +51,17 @@
         'numpy!=1.24.3',  # severe masking bug
         'openpyxl',
         'packaging',
-        'pandas!=2.2.0,!=2.2.1,!=2.2.2',  # ESMValCore PR2305
+        'pandas==2.1.4',  # see note in environment.yml
         'progressbar2',
-        'psyplot',
-        'psy-maps',
-        'psy-reg',
-        'psy-simple',
+        'psyplot>=1.5.0',  # psy*<1.5.0 are not py312 compat
+        'psy-maps>=1.5.0',
+        'psy-reg>=1.5.0',
+        'psy-simple>=1.5.0',
         'pyproj>=2.1',
         'pys2index',
         'python-dateutil',
         'pyyaml',
-        'rasterio',
+        'rasterio>=1.3.10',
         'requests',
         'ruamel.yaml',
         'scikit-image',
@@ -104,7 +104,7 @@
         'imagehash',
         'isort',
         'pre-commit',
-        'prospector[with_pyroma]!=1.1.6.3,!=1.1.6.4',
+        'prospector[with_pyroma]>=1.12',
         'vprof',
         'yamllint',
         'yapf',
@@ -224,6 +224,7 @@ def read_description(filename):
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: Scientific/Engineering',
         'Topic :: Scientific/Engineering :: Atmospheric Science',
         'Topic :: Scientific/Engineering :: GIS',