Merge branch 'main' into cron_esmvalbot_test

.circleci/config.yml

@@ -158,7 +158,7 @@ jobs:
   test_installation_from_source_test_mode:
     # Test installation from source
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -167,7 +167,7 @@ jobs:
   test_installation_from_source_develop_mode:
     # Test development installation
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -179,7 +179,7 @@ jobs:
     # purpose of this test to discover backward-incompatible changes early on in
     # the development cycle.
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - run:
@@ -216,8 +216,8 @@ jobs:
             conda activate esmvaltool
             mkdir -p ~/climate_data
             esmvaltool config get_config_user
-            echo "search_esgf: when_missing" >> ~/.esmvaltool/config-user.yml
-            cat ~/.esmvaltool/config-user.yml
+            echo "search_esgf: when_missing" >> ~/.config/esmvaltool/config-user.yml
+            cat ~/.config/esmvaltool/config-user.yml
             for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
               esmvaltool run "$recipe"
             done
@@ -233,7 +233,7 @@ jobs:
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: medium
     steps:
       - checkout
@@ -257,7 +257,7 @@ jobs:
   test_installation_from_conda:
     # Test conda package installation
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/miniforge3:latest
     resource_class: large
     steps:
       - run:

.github/workflows/citation_file_validator.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/[email protected]

.github/workflows/create-condalock-file.yml

@@ -20,24 +20,25 @@ jobs:
     name: Create conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.11"
+          python-version: "3.12"
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - name: Show conda config
         run: |
           conda update -n base -c conda-forge conda
           conda --version
-          # setup-miniconda@v2 installs an old conda and mamba
+          # setup-miniconda@v3 installs an old conda and mamba
           # forcing a modern mamba updates both mamba and conda
-          conda install -c conda-forge "mamba>=1.4.8"
+          # unpin mamba after conda-lock=3 release
+          # see github.com/ESMValGroup/ESMValTool/issues/3782
+          conda install -c conda-forge "mamba>=1.4.8,<2"
           conda config --show-sources
           conda config --show
           conda --version

.github/workflows/install-from-conda.yml

@@ -20,14 +20,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
@@ -44,7 +43,7 @@ jobs:
           esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_Linux_python_${{ matrix.python-version }}
           path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -56,12 +55,12 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.9", "3.10", "3.11"]
+#        python-version: ["3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:
 #      - uses: actions/checkout@v2
-#      - uses: conda-incubator/setup-miniconda@v2
+#      - uses: conda-incubator/setup-miniconda@v3
 #        with:
 #          python-version: ${{ matrix.python-version }}
 #          miniconda-version: "latest"
@@ -82,7 +81,7 @@ jobs:
 #      - run: esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
 #      - name: Upload artifacts
 #        if: ${{ always() }}  # upload artifacts even if fail
-#        uses: actions/upload-artifact@v2
+#        uses: actions/upload-artifact@v4
 #        with:
 #          name: Conda_Install_OSX_python_${{ matrix.python-version }}
 #          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}

.github/workflows/install-from-condalock-file.yml

@@ -30,14 +30,14 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool-fromlock
           python-version: ${{ matrix.python-version }}
@@ -57,7 +57,7 @@ jobs:
       - run: pytest -n 2 -m "not installation"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Source_Install_Linux_python_${{ matrix.python-version }}
           path: source_install_linux_artifacts_python_${{ matrix.python-version }}

.github/workflows/install-from-source.yml

@@ -19,20 +19,19 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool
           environment-file: environment.yml
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
@@ -47,7 +46,7 @@ jobs:
           esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Source_Install_Linux_python_${{ matrix.python-version }}
           path: source_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -59,12 +58,12 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.9", "3.10", "3.11"]
+#        python-version: ["3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:
 #      - uses: actions/checkout@v2
-#      - uses: conda-incubator/setup-miniconda@v2
+#      - uses: conda-incubator/setup-miniconda@v3
 #        with:
 #          activate-environment: esmvaltool
 #          environment-file: environment.yml
@@ -79,7 +78,7 @@ jobs:
 #      - run: esmvaltool version 2>&1 | tee source_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
 #      - name: Upload artifacts
 #        if: ${{ always() }}  # upload artifacts even if fail
-#        uses: actions/upload-artifact@v2
+#        uses: actions/upload-artifact@v4
 #        with:
 #          name: Source_Install_OSX_python_${{ matrix.python-version }}
 #          path: source_install_osx_artifacts_python_${{ matrix.python-version }}

.github/workflows/pypi-build-and-deploy.yml

@@ -14,13 +14,13 @@ jobs:
     name: Build and publish ESMValTool on PyPi
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install pep517
         run: >-
           python -m

.github/workflows/run-tests-monitor.yml

@@ -23,25 +23,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool
           environment-file: environment.yml
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
         run: |
           mamba --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
           python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Inspect environment
+        run: conda list
       - name: Install pytest-monitor
         run: pip install pytest-monitor
       - name: Install ESMValTool
@@ -56,7 +57,7 @@ jobs:
         run: python tests/parse_pymon.py
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Test_Linux_python_${{ matrix.python-version }}
           path: test_linux_artifacts_python_${{ matrix.python-version }}
@@ -65,20 +66,21 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
+          architecture: ${{ matrix.architecture }}
           activate-environment: esmvaltool
           environment-file: environment_osx.yml
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       # - name: Install libomp with homebrew
       #   run: brew install libomp
@@ -87,6 +89,10 @@ jobs:
         run: |
           mamba --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
           python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Inspect environment
+        run: conda list
+      - name: Install git
+        run: mamba install -c conda-forge git
       - name: Install pytest-monitor
         run: pip install pytest-monitor
       - name: Install ESMValTool
@@ -101,7 +107,7 @@ jobs:
         run: python tests/parse_pymon.py
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Test_OSX_python_${{ matrix.python-version }}
           path: test_osx_artifacts_python_${{ matrix.python-version }}

.github/workflows/test-development.yml

@@ -12,7 +12,6 @@ on:
   push:
     branches:
       - main
-      - fix_recipe_filler_bkwds_incompatibility
   schedule:
     - cron: '0 0 * * *'
 
@@ -27,19 +26,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool
           environment-file: environment.yml
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p develop_test_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
@@ -62,7 +60,7 @@ jobs:
         run: pytest -n 2 -m "not installation" 2>&1 | tee develop_test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Develop_Test_Linux_python_${{ matrix.python-version }}
           path: develop_test_linux_artifacts_python_${{ matrix.python-version }}