Update development infrastructure (#2663)

Closes #2606 Closes #2299 Workaround for #2695 Closes ESMValGroup/ESMValCore#1364? Versioning Use setuptools-scm to give a more accurate version number. The version number is built up as follows: {major}.{minor}.{patch}.dev{distance}+g{commit}+d{date}. Examples: For released versions, it will remain unchanged, e.g. 2.5.0 If there are additional commits since the release, the patch version number will be increased with one and the number of commits since the release and the current commit will be used to generate a more detailed description. For example, if you are on v2.5.0 of the tool and there have been 72 commits in your current branch since that release and the latest commit has the (hexadecimal) number a424c89 (you can find the number of all commits in the current branch by using git log), then the version number will look like this: 2.5.1.dev72+ga424c891d. If there are uncommitted changes, the letter d and the current date will be added to the version number, e.g. 2.5.1.dev72+ga424c891d.d20220719 CircleCI Use appropriate resources Upload compressed artifacts for fast runs Upload test results in a format that CircleCI understands, enabling test summaries and statistics Only run documentation test nightly, readthedocs build is now configured to fail on warnings Readthedocs Use mamba build Fail on warnings Docker Use mamba container as a basis Python development installation Install all development dependencies from conda-forge R development installation Install all development dependencies from conda-forge Stickler CI Remove the configuration file because we are not using this service anymore Updates to dependencies Pin cf-units to a version that is compatible with the tests for the v2.6 release (Test failure due to change in cf-units ESMValCore#1655) Pin r-akima (recipe_miles_* broken due to newest akima package versions #2695) Pin xesfm to 0.3.0 (Old xesmf version dependency creates problems #2729) Remove xesmf dependency numba (Dependency refinement sparse, numba pangeo-data/xESMF#158)
ESMValGroup · Jul 21, 2022 · 487d878 · 487d878
1 parent 99ea8b6
commit 487d878
Show file tree

Hide file tree

Showing 29 changed files with 399 additions and 529 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -7,174 +7,189 @@ orbs:
 commands:
   check_changes:
     steps:
-      - run: |
-          if (test "$CIRCLE_BRANCH" = main ||
-            git --no-pager diff --name-only origin/main... |
-            grep -q -E -f .circleci/install_triggers)
-          then
-            echo Running installation tests
-          else
-            echo Skipping installation tests
-            circleci step halt
-          fi
-
-jobs:
-  run_tests:
-    # Run Python 3 tests
-    working_directory: /test
-    docker:
-      - image: esmvalgroup/esmvaltool:development
+      - run:
+          name: Check whether or not installation tests are needed
+          command: |
+            if (test "$CIRCLE_BRANCH" = main ||
+              git --no-pager diff --name-only origin/main... |
+              grep -q -E -f .circleci/install_triggers)
+            then
+              echo Running installation tests
+            else
+              echo Skipping installation tests
+              circleci step halt
+            fi
+  test_and_report:
     steps:
-      - checkout
       - run:
-          # Update environment if needed and run tests
+          name: Run tests
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            conda --version > /logs/conda_version.txt
             conda activate esmvaltool
-            python --version > /logs/python_version.txt
-            conda env export > /logs/environment.yml
-            pip install .[test] > /logs/install.txt
-            # Run flake8 check with source code (it doesn't work without it)
-            pytest -n 2 -m flake8
-            # Update/install Julia dependencies
-            esmvaltool install Julia > /logs/install_Julia.txt
-            # Remove source to test installed software
-            rm -r esmvaltool
-            pytest -n 2 -m 'not flake8'
+            pytest -n 4 -m 'not flake8' --junitxml=test-reports/report.xml
+            esmvaltool version
+            esmvaltool -- --help
+            ncl -V
+            cdo --version
+      - store_test_results:
+          path: test-reports/
       - store_artifacts:
           path: /logs
+      - run:
+          name: Compress pytest artifacts
+          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-root/ .
+          when: always
       - store_artifacts:
-          path: test-reports/
-      - store_test_results:
-          path: test-reports/
-      - coverage-reporter/send_report:
-          coverage-reports: 'test-reports/coverage.xml'
-          project-token: $CODACY_PROJECT_TOKEN
-          # Skip if token isn't defined.
-          # Useful to let forks CI pass without passing secrets
-          skip: true
-
-  test_installation_from_source_test_mode:
-    # Test Python 3 installation
-    working_directory: /test_installation
-    docker:
-      - image: condaforge/mambaforge:latest
+          path: pytest.tar.gz
+      - run:
+          name: Compress test-report artifacts
+          command: tar -cvzf test-reports.tar.gz test-reports/
+          when: always
+      - store_artifacts:
+          path: test-reports.tar.gz
+  test_installation_from_source:
+    parameters:
+      extra:
+        type: string
+      flags:
+        type: string
+        default: ""
     steps:
+      - run:
+          name: Install git+ssh
+          environment:
+            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+          command: apt update && apt install -y git ssh
       - checkout
       - check_changes
+      - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
       - restore_cache:
-          key: test-install-{{ .Branch }}
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
+          name: Install dependencies
           command: |
+            # Install
             . /opt/conda/etc/profile.d/conda.sh
-            set -x
             mkdir /logs
-            # Install
-            mamba create --name esmvaltool 'python=3.10' >> /logs/conda.txt 2>&1
-            mamba env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x; conda activate esmvaltool; set -x
-            mamba install julia >> /logs/conda.txt 2>&1
-            pip install .[test] > /logs/install.txt 2>&1
-            esmvaltool install Julia
-            # Remove source to test installed software
-            rm -r esmvaltool
-            # Log versions
+            mamba env create >> /logs/conda.txt 2>&1
+            conda activate esmvaltool
+            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            if [[ "<<parameters.flags>>" != *'--editable'* ]]
+            then
+              rm -r esmvaltool
+            fi
+      - run:
+          name: Log versions
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
             dpkg -l > /logs/versions.txt
-            mamba env export > /logs/environment.yml
+            conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
-            # Test installation
-            pytest -n 2
-            esmvaltool -h
-            ncl -V
-            # cdo test, check that it supports hdf5
-            cdo --version
-            echo 0 | cdo -f nc input,r1x1 tmp.nc
-            ncdump tmp.nc | ncgen -k hdf5 -o tmp.nc
-            cdo -f nc copy tmp.nc tmp2.nc
-          no_output_timeout: 30m
+      - run:
+          name: Lint source code
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            pytest -n 4 -m flake8
+      - test_and_report
       - save_cache:
-          key: test-install-{{ .Branch }}
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
-            - "/opt/conda/pkgs"
-            - ".eggs"
-      - store_artifacts:
-          path: /logs
-      - store_artifacts:
-          path: test-reports/
-      - store_test_results:
-          path: test-reports/
+            - /opt/conda/pkgs
+            - /root/.cache/pip
+            - .pytest_cache
 
-  test_installation_from_source_develop_mode:
-    # Test development installation
-    working_directory: /develop
+jobs:
+  run_tests:
+    # Run tests
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: esmvalgroup/esmvaltool:development
+    resource_class: large
     steps:
       - checkout
-      - check_changes
       - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
+      - restore_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install dependencies
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            set -x
             mkdir /logs
-            # Install
-            mamba create --name esmvaltool 'python=3.10' >> /logs/conda.txt 2>&1
-            mamba env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x; conda activate esmvaltool; set -x
-            mamba install julia >> /logs/conda.txt 2>&1
-            pip install -e .[develop] > /logs/install.txt 2>&1
-            esmvaltool install Julia
-            # Log versions
-            dpkg -l > /logs/versions.txt
-            mamba env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
-            # Test installation
-            pytest -n 2
-            esmvaltool -h
-            ncl -V
-            cdo --version
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: /logs
+            conda activate esmvaltool
+            pip install .[test] > /logs/install.txt 2>&1
+            # Run flake8 check with source code (it doesn't work without it)
+            pytest -n 4 -m flake8
+            # Update/install Julia dependencies
+            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            # Remove source to test installed software
+            rm -r esmvaltool
+      - test_and_report
+      - save_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /root/.cache/pip
+            - .pytest_cache
+      - coverage-reporter/send_report:
+          coverage-reports: 'test-reports/coverage.xml'
+          project-token: $CODACY_PROJECT_TOKEN
+          skip: true  # skip if project-token is not defined (i.e. on a fork)
+
+  test_installation_from_source_test_mode:
+    # Test installation from source
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          extra: test
+
+  test_installation_from_source_develop_mode:
+    # Test development installation
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          extra: develop
+          flags: "--editable"
 
   build_documentation:
     # Test building documentation
-    working_directory: /doc
     docker:
-      - image: condaforge/mambaforge:latest
+      - image: condaforge/mambaforge
+    resource_class: small
     steps:
       - checkout
-      - restore_cache:
-          key: documentation-{{ .Branch }}
       - run:
           command: |
             mkdir /logs
+            . /opt/conda/etc/profile.d/conda.sh
             # Install
-            pip install -r doc/sphinx/source/requirements.txt > /logs/install.txt 2>&1
-            pip install sphinx sphinx_rtd_theme >> /logs/install.txt 2>&1
+            mamba env create
+            conda activate esmvaltool
+            pip install .[doc]
             # Log versions
             dpkg -l > /logs/versions.txt
+            conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test building documentation
-            MPLBACKEND=Agg sphinx-build -W doc/sphinx/source/ doc/sphinx/build/
-      - save_cache:
-          key: documentation-{{ .Branch }}
-          paths:
-            - ".eggs"
+            MPLBACKEND=Agg sphinx-build -W doc/sphinx/source doc/spinx/build
       - store_artifacts:
           path: /logs
 
   test_installation_from_conda:
     # Test conda package installation
-    working_directory: /esmvaltool
     docker:
       - image: condaforge/mambaforge
+    resource_class: medium
     steps:
       - run:
-          environment:
-            R_INSTALL_N_CPUS: 2
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             set -x
@@ -184,15 +199,15 @@ jobs:
             mamba create -y --name esmvaltool -c conda-forge esmvaltool julia 'python=3.10' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
-            # install the package
-            mamba install -c conda-forge esmvaltool >> /logs/install.txt 2>&1
+            # install the Julia dependencies
+            esmvaltool install Julia > /logs/install_Julia.txt 2>&1
             # Log versions
             mamba env export > /logs/environment.yml
             # Test installation
-            esmvaltool -h
+            esmvaltool -- --help
+            esmvaltool version
             ncl -V
             cdo --version
-          no_output_timeout: 30m
       - store_artifacts:
           path: /logs
 
@@ -202,8 +217,6 @@ workflows:
       - run_tests
       - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
-      - build_documentation
-      - test_installation_from_conda
   nightly:
     triggers:
       - schedule:

diff --git a/.dockerignore b/.dockerignore
@@ -2,10 +2,9 @@
 **/__pycache__
 .*
 doc
-package
 tests
 ESMValTool.egg-info
 
-
+!.git
 !.zenodo.json
 
diff --git a/.github/workflows/citation_file_validator.yml b/.github/workflows/citation_file_validator.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/[email protected]

diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -3,7 +3,7 @@ name: Conda lock file creation
 on:
   # Trigger on push on main or other branch for testing
   # NOTE that push: main will create the file very often
-  # and hence lots of automated PRs 
+  # and hence lots of automated PRs
   # push:
   #   branches:
   #     - condalock-update
@@ -15,7 +15,9 @@ jobs:
     name: Create conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true

diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml
@@ -18,7 +18,6 @@ jobs:
         python-version: ["3.8", "3.9", "3.10"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v2
       - uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python-version }}

diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -29,7 +29,9 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: esmvaltool-fromlock