Merge pull request #365 from Climate-REF/self-hosted

Author: lewisjared

.github/workflows/ci-integration.yaml

@@ -13,16 +13,15 @@ on:
 
 jobs:
   tests-slow:
+    if: github.repository == 'Climate-REF/climate-ref'
     env:
       REF_TEST_OUTPUT: "test-outputs"
-      REF_DATASET_CACHE_DIR: "data"
       PYTEST_ADDOPTS: "--slow"
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-latest" ]
-        python-version: [ "3.11", "3.12", "3.13" ]
-    runs-on: "${{ matrix.os }}"
+        python-version: [ "3.11", "3.13" ]
+    runs-on: "self-hosted"
     defaults:
       run:
         shell: bash
@@ -32,21 +31,15 @@ jobs:
       - uses: ./.github/actions/setup
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Cache downloaded data
-        uses: actions/cache@v4
-        with:
-          path: |
-            ${{ env.REF_DATASET_CACHE_DIR }}
-          key: reference-datasets
       - name: Run tests
         run: |
           make virtual-environment
           make fetch-test-data
-          uv run ref datasets fetch-data --registry ilamb
-          uv run ref datasets fetch-data --registry iomb
-          uv run ref datasets fetch-data --registry esmvaltool
+          uv run ref datasets fetch-data --registry ilamb --symlink
+          uv run ref datasets fetch-data --registry iomb --symlink
+          uv run ref datasets fetch-data --registry esmvaltool --symlink
           uv run ref providers create-env
-          make test
+          uv run pytest packages tests --slow --no-docker -r a -v
       # Upload the scratch and executions directories as artifacts
       - name: Upload scratch artifacts
         uses: actions/upload-artifact@v4

changelog/365.improvement.md

@@ -0,0 +1,3 @@
+Use a self-hosted CI runner for the integration tests.
+The GitHub runner do not have enough disk space to store the required datasets,
+and these data are downloaded over the internet on each run.

conftest.py

@@ -42,10 +42,12 @@
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: mark test as slow to run")
+    config.addinivalue_line("markers", "docker: mark test requires docker to run")
 
 
 def pytest_addoption(parser):
     parser.addoption("--slow", action="store_true", help="include tests marked slow")
+    parser.addoption("--no-docker", action="store_true", help="skip docker tests")
 
 
 def pytest_collection_modifyitems(config, items):
@@ -54,6 +56,11 @@ def pytest_collection_modifyitems(config, items):
         for item in items:
             if item.get_closest_marker("slow"):
                 item.add_marker(skip_slow)
+    if config.getoption("--no-docker"):
+        skip_docker = pytest.mark.skip(reason="--no-docker option provided")
+        for item in items:
+            if item.get_closest_marker("docker"):
+                item.add_marker(skip_docker)
 
 
 @pytest.fixture(scope="session")

packages/climate-ref-core/pyproject.toml

@@ -39,7 +39,7 @@ dependencies = [
     "environs>=11",
     "pyyaml>=6.0.2",
     # Not used directly, but required to support some installations
-    "setuptools>=75.8.0",
+    "setuptools<81",
 
     # SPEC 0000 constraints
     # We follow [SPEC-0000](https://scientific-python.org/specs/spec-0000/)

tests/integration/test_postgres.py

@@ -75,6 +75,7 @@ def config(config, postgres_container):
     return config
 
 
+@pytest.mark.docker
 def test_connect_and_migrations(config, cmip6_data_catalog):
     database = Database.from_config(config)
     assert database.url.startswith("postgresql")
@@ -87,6 +88,7 @@ def test_connect_and_migrations(config, cmip6_data_catalog):
             adapter.register_dataset(config, database, data_catalog_dataset)
 
 
+@pytest.mark.docker
 def test_check_up_to_date(config):
     database = Database.from_config(config)