fixes issue with UPath 0.3.x (#1374)

Author: normanrz

.github/workflows/ci.yml

@@ -176,7 +176,6 @@ jobs:
         timeout-minutes: 30
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
-          COVERAGE_FILE: "~/.local/share/coverage/${{ matrix.group }}.coverage"
         run: ./test.sh --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration --cov=webknossos --cov-report=
 
       - name: Python tests
@@ -191,13 +190,13 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: coverage-report-${{ matrix.group }}
-          path: "~/.local/share/coverage/${{ matrix.group }}.coverage"
+          path: "webknossos/.coverage"
+          include-hidden-files: true
+          if-no-files-found: error
           retention-days: 1 # Automatically deletes after 1 day
 
       - name: Check if git is dirty
-        run: |
-          git diff --no-ext-diff --exit-code
-          [[ -z $(git status -s) ]]
+        run: ../_tooling/check_clean.sh
 
   webknossos_windows:
     needs: changes
@@ -262,7 +261,7 @@ jobs:
         env:
           COVERAGE_FILE: "~/coverage-files/result.coverage"
         run: |
-          coverage combine ~/coverage-files/**/*.coverage
+          coverage combine ~/coverage-files/**/.coverage
           coverage xml -i -o ~/coverage-files/result.xml
 
       - name: Report coverage
@@ -301,8 +300,8 @@ jobs:
           cd docs
           ./generate.sh --persist
 
-      - name: Push docs (for branch)
-        if: github.ref_type == 'branch'
+      - name: Push docs (for master)
+        if: github.ref == 'refs/heads/master'
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

_tooling/check_clean.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -eEuo pipefail
+
+echo "🔍 Checking for uncommitted changes..."
+
+# Include both committed and untracked changes
+if ! git diff --quiet HEAD || [ -n "$(git ls-files --others --exclude-standard)" ]; then
+  echo "❌ Git working tree is dirty:"
+  git status --short
+  exit 1
+fi
+
+echo "✅ Working tree clean."

cluster_tools/pyproject.toml

@@ -21,8 +21,8 @@ dask = ["distributed ~=2023.9.1"]
 kubernetes = ["kubernetes ~=27.2.0"]
 all=["cluster_tools[kubernetes]", "cluster_tools[dask]"]
 
-[tool.uv]
-dev-dependencies = [
+[dependency-groups]
+dev = [
     "icecream ~=2.1.1",
     "mypy ~=1.15.0",
     "pytest ~=8.3.3",

webknossos/Changelog.md

@@ -29,6 +29,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 - Dataset.add_layer_as_ref() now accepts RemoteLayer objects as well as Layer objects. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
 
 ### Changed
+- Replaced `pathlib.Path` for `upath.UPath` internally. `pathlib.Path` is still supported for user-facing APIs. Adds support for `universal_pathlib` version 0.3.x. [#1374](https://github.com/scalableminds/webknossos-libs/pull/1374)
 - Only use fs-based copy for mags on local file systems and S3. Other protocols, e.g. memory and HTTP, have caveats that break fs-based copying. [#1365](https://github.com/scalableminds/webknossos-libs/pull/1365)
 - The `add_*` methods in `Attachments` now return the created attachment objects, similar to `add_layer` and `add_mag`. [#1365](https://github.com/scalableminds/webknossos-libs/pull/1365)
 - The returned dataset ID from dataset exploration is now used. [#1378](https://github.com/scalableminds/webknossos-libs/pull/1378) 

webknossos/pyproject.toml

@@ -51,10 +51,11 @@ dependencies = [
     "rich ~=13.8.0",
     "s3fs>=2024.6.1",
     "scipy >=1.13",
-    "tensorstore~=0.1.74",
+    "tensorstore ==0.1.76; sys_platform == 'win32'", # tensorstore crashes on windows with >0.1.76
+    "tensorstore ~=0.1.78; sys_platform != 'win32'",
     "typer ~=0.12.5",
     "typing-extensions ~=4.0",
-    "universal-pathlib ~=0.2",
+    "universal-pathlib ~=0.3",
     "wkw ==1.1.24",
     "zipp ~=3.5.0",
 ]
@@ -89,9 +90,8 @@ all = [
     "webknossos[examples]",
 ]
 
-[tool.uv]
-extra-index-url = ["https://pypi.scm.io/simple"]
-dev-dependencies = [
+[dependency-groups]
+dev = [
     "coverage>=7.8.0",
     "fsspec ~=2024.6.0",
     "hypothesis ~=6.35.0",
@@ -110,6 +110,9 @@ dev-dependencies = [
     "ruff ~=0.11.0",
 ]
 
+[tool.uv]
+extra-index-url = ["https://pypi.scm.io/simple"]
+
 [tool.uv.sources]
 cluster-tools = { path = "../cluster_tools", editable = true }
 

webknossos/script_collection/test_segmentation_heuristic.py

@@ -1,5 +1,6 @@
 import sys
-from pathlib import Path
+
+from upath import UPath
 
 import webknossos as wk
 from webknossos.dataset._utils.segmentation_recognition import (
@@ -9,11 +10,11 @@
 )
 
 
-def looks_like_remote_dataset(path: Path) -> bool:
+def looks_like_remote_dataset(path: UPath) -> bool:
     return len([item for item in path.iterdir() if item.is_dir()]) == 0
 
 
-def main(path_to_datasets: Path) -> None:
+def main(path_to_datasets: UPath) -> None:
     wrongly_predicted: list[str] = []
     correctly_predicted: list[str] = []
     failures: list[str] = []
@@ -79,4 +80,4 @@ def main(path_to_datasets: Path) -> None:
 
 
 if __name__ == "__main__":
-    main(Path("../webknossos/binaryData/sample_organization/"))
+    main(UPath("../webknossos/binaryData/sample_organization/"))

webknossos/test.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -eEuo pipefail
+set -x
 
 source local_wk_setup.sh
 

webknossos/tests/conftest.py

@@ -2,9 +2,8 @@
 import os
 import warnings
 from collections.abc import Generator, Iterator
-from os import makedirs
 from pathlib import Path
-from shutil import rmtree, unpack_archive
+from shutil import unpack_archive
 from typing import Any
 
 import fsspec.implementations.http as http
@@ -16,6 +15,7 @@
 import webknossos as wk
 from webknossos.client._upload_dataset import _cached_get_upload_datastore
 from webknossos.client.context import _clear_all_context_caches
+from webknossos.utils import rmtree
 
 from .constants import TESTDATA_DIR, TESTOUTPUT_DIR
 
@@ -98,7 +98,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
 
 @pytest.fixture(autouse=True, scope="function")
 def clear_testoutput() -> Generator:
-    makedirs(TESTOUTPUT_DIR, exist_ok=True)
+    TESTOUTPUT_DIR.mkdir(parents=True, exist_ok=True)
     yield
     rmtree(TESTOUTPUT_DIR)
 
@@ -145,12 +145,9 @@ def use_replay_proxay(request: Any) -> Generator:
 
 
 @pytest.fixture(scope="session")
-def WT1_path() -> Path:
+def WT1_upath() -> UPath:
     ds_path = TESTDATA_DIR / "WT1_wkw"
     if ds_path.exists():
         rmtree(ds_path)
-    unpack_archive(
-        TESTDATA_DIR / "WT1_wkw.tar.gz",
-        ds_path,
-    )
+    unpack_archive(str(TESTDATA_DIR / "WT1_wkw.tar.gz"), str(ds_path))
     return ds_path