refactor dataset, remote_dataset, and layer

docs/mkdocs.yml

@@ -245,9 +245,9 @@ nav:
         - webknossos-py/examples/download_tiff_stack.md
         - webknossos-py/examples/remote_datasets.md
         - webknossos-py/examples/convert_4d_tiff.md
-        - webknossos-py/examples/announce_dataset_upload.md
         - webknossos-py/examples/accessing_metadata.md
         - webknossos-py/examples/explore_and_add_remote.md
+        - webknossos-py/examples/upload_to_webknossos.md
       - Annotation Examples:
         - webknossos-py/examples/apply_merger_mode.md
         - webknossos-py/examples/learned_segmenter.md

docs/src/webknossos-py/examples/upload_to_webknossos.md

@@ -0,0 +1,9 @@
+# Upload dataset to WEBKNOSSOS
+
+This example demonstrates the process of uploading a Dataset. 
+
+```python
+--8<--
+webknossos/examples/upload_to_webknossos.py
+--8<--
+```

webknossos/Changelog.md

@@ -14,10 +14,22 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 
 ### Breaking Changes
 - Removed Docker image `scalableminds/webknossos-cli` build. It will not be released further. [#1376](https://github.com/scalableminds/webknossos-libs/pull/1376)
+- `Dataset.add_layer_as_ref(remote_layer.path)` does not work anymore. Please use `Dataset.add_layer_as_ref(remote_layer)` instead. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
+- Due to the refactoring, the imports of various classes have changed. Please update your code accordingly. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
+- Deprecated a number of methods [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371]):
+  - `Dataset.open_remote()`, use `RemoteDataset.open` instead.
+  - `RemoteDatset.download_mesh()`, use `remote_segmentation_layer.download_mesh()` instead.
+  - `Dataset.get_remote_datasets()`, use `RemoteDataset.list()` instead.
+  - `Dataset.download()`, use `RemoteDataset.download()` instead.
+  - `mag_view.is_foreign`, use `layer.is_mag_view_foreign(mag_view)` instead.
+  - `Dataset.trigger_reload_in_datastore()`, use `RemoteDataset.trigger_reload_in_datastore()` instead.
 
 ### Added
+- Dataset.add_layer_as_ref() now accepts RemoteLayer objects as well as Layer objects. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
 
 ### Changed
+- Refactored the architecture, by introducing RemoteLayers, RemoteSegmentationLayers and their abstract base classes. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
+- Updated the api version of the webknossos-api to 12. [#1371](https://github.com/scalableminds/webknossos-libs/pull/1371])
 
 ### Fixed
 

webknossos/examples/accessing_metadata.py

@@ -2,7 +2,7 @@
 
 
 def main() -> None:
-    l4_sample_dataset = wk.Dataset.open_remote("l4_sample")
+    l4_sample_dataset = wk.RemoteDataset.open("l4_sample")
     # Access the metadata of the dataset
     print(l4_sample_dataset.metadata)
 

webknossos/examples/apply_merger_mode.py

@@ -1,11 +1,7 @@
-from pathlib import Path
-
 import fastremap
 
 import webknossos as wk
 
-TESTDATA_DIR = Path(__file__).parent.parent / "testdata"
-
 
 def main() -> None:
     ####################################
@@ -18,15 +14,14 @@ def main() -> None:
     ).skeleton
 
     ###############################################
-    # Download and open the corresponding dataset #
+    # Open the corresponding dataset #
     ###############################################
 
-    dataset = wk.Dataset.download(
+    input_dataset = wk.RemoteDataset.open(
         "l4_sample",
-        bbox=wk.BoundingBox((3457, 3323, 1204), (40, 10, 10)),
-        path="testoutput/l4_sample",
+        organization_id="scalable_minds",
     )
-    in_layer = dataset.get_segmentation_layer("segmentation")
+    in_layer = input_dataset.get_segmentation_layer("segmentation")
     in_mag1 = in_layer.get_mag("1")
 
     ##############################
@@ -52,8 +47,10 @@ def main() -> None:
     ############################
     # Creating an output layer #
     ############################
-
-    out_layer = dataset.add_layer(
+    output_dataset = wk.Dataset(
+        "testoutput/l4_sample", voxel_size=input_dataset.voxel_size
+    )
+    out_layer = output_dataset.add_layer(
         "segmentation_remapped",
         wk.SEGMENTATION_CATEGORY,
         dtype_per_channel=in_layer.dtype_per_channel,
@@ -84,16 +81,9 @@ def apply_mapping_for_chunk(args: tuple[wk.View, wk.View, int]) -> None:
     ########################################
 
     out_layer.downsample()
-    dataset.delete_layer("segmentation")
-    dataset.upload(
+    output_dataset.upload(
         "l4_sample_remapped",
-        layers_to_link=[
-            wk.LayerToLink(
-                dataset_name="l4_sample",
-                layer_name="color",
-                organization_id="scalable_minds",
-            )
-        ],
+        layers_to_link=[input_dataset.get_layer("color")],
     )
 
 

webknossos/examples/create_dataset_from_images.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
-from webknossos import Dataset
-from webknossos.dataset import COLOR_CATEGORY
+from webknossos import COLOR_CATEGORY, Dataset
 
 INPUT_DIR = Path(__file__).parent.parent / "testdata" / "tiff"
 OUTPUT_DIR = Path("testoutput/my_tiff_dataset")

webknossos/examples/dataset_usage.py

@@ -9,7 +9,6 @@ def main() -> None:
     #####################
     # Opening a dataset #
     #####################
-
     dataset = wk.Dataset.open("testdata/simple_wkw_dataset")
     # Assuming that the dataset has a layer "color"
     # and the layer has the magnification 1
@@ -67,7 +66,7 @@ def main() -> None:
 
     copy_of_dataset = dataset.copy_dataset(
         "testoutput/copy_of_dataset",
-        chunk_shape=(8, 8, 8),
+        chunk_shape=(32, 32, 32),
         shard_shape=(64, 64, 64),
         compress=True,
     )

webknossos/examples/download_image_data.py

@@ -15,13 +15,14 @@ def main() -> None:
     bbox = wk.BoundingBox([50, 50, 50], [50, 50, 50])
 
     # Download the dataset
-    ds = wk.Dataset.download(
+    ds = wk.RemoteDataset.open(
         dataset_name_or_url=name,
         organization_id=organization_id,
+        webknossos_url="https://webknossos.org",
+    ).download(
         bbox=bbox,
         layers=layers,
         mags=mags,
-        webknossos_url="https://webknossos.org",
     )
 
     # Check that the layer was downloaded

webknossos/examples/download_tiff_stack.py

@@ -8,7 +8,7 @@
 
 
 def main() -> None:
-    dataset = wk.Dataset.open_remote(
+    dataset = wk.RemoteDataset.open(
         DATASET_NAME,
         organization_id="scalable_minds",
         webknossos_url="https://webknossos.org",

webknossos/examples/image_stack_to_dataset.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 
-from webknossos import Dataset
-from webknossos.dataset import COLOR_CATEGORY
+from webknossos import COLOR_CATEGORY, Dataset
 
 INPUT_FILES = (
     Path(__file__).parent.parent / "testdata" / "tiff" / "test.0000.tiff",

webknossos/examples/remote_datasets.py

@@ -5,7 +5,7 @@
 
 def main() -> None:
     # Remote datasets are read-only, but can be used similar to normal datasets:
-    l4_sample_dataset = wk.Dataset.open_remote("l4_sample")
+    l4_sample_dataset = wk.RemoteDataset.open("l4_sample")
 
     # Print information of the public l4dense_motta_et_al_demo dataset:
     print(l4_sample_dataset.url)
@@ -14,7 +14,7 @@ def main() -> None:
     print("Tags:", ", ".join(l4_sample_dataset.tags))
 
     # List all accessible remote datasets via get_remote_datasets():
-    own_remote_datasets = wk.Dataset.get_remote_datasets()
+    own_remote_datasets = wk.RemoteDataset.list()
 
     # Print the first 10 dataset names from your organization:
     print()
@@ -26,7 +26,7 @@ def main() -> None:
     l4_sample_dataset.tags = ("demo",)
 
     # List all accessible demo datasets:
-    remote_demo_datasets = wk.Dataset.get_remote_datasets(tags="demo")
+    remote_demo_datasets = wk.RemoteDataset.list(tags="demo")
     print("Remote demo datasets:", list(remote_demo_datasets))
     assert l4_sample_dataset in remote_demo_datasets.values()
 

webknossos/examples/upload_image_data.py

@@ -2,8 +2,8 @@
 from skimage import data
 
 import webknossos as wk
-from webknossos.dataset import COLOR_CATEGORY
-from webknossos.dataset.properties import (
+from webknossos import COLOR_CATEGORY
+from webknossos.dataset_properties import (
     DatasetViewConfiguration,
     LayerViewConfiguration,
 )

webknossos/examples/upload_to_webknossos.py

@@ -0,0 +1,30 @@
+import webknossos as wk
+from webknossos import LayerToLink
+from webknossos.dataset.remote_folder import RemoteFolder
+
+
+def main() -> None:
+    # Get the folder id of the upload destination:
+    folder_id = RemoteFolder.get_by_path("Datasets/").id
+
+    # Open your unpublished dataset
+    dataset = wk.Dataset.open("my_dataset.wkw")
+
+    existing_dataset = wk.RemoteDataset.open(
+        "https://webknossos.org/datasets/liconn_expid82-678f98e3010000dd017f4449/#3825,4866,795,0,1.3"
+    )
+    # Publish the dataset to webknossos
+    # The data will be uploaded to the upload destination folder
+    dataset.upload(
+        new_dataset_name="my_new_dataset_name",
+        folder_id=folder_id,
+        layers_to_link=[
+            LayerToLink.from_remote_layer(
+                existing_dataset.get_layer("image_230130b"), new_layer_name="image"
+            ),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    main()

webknossos/pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
     "python-dateutil ~=2.8.0",
     "python-dotenv ~=1.0.1",
     "rich ~=13.8.0",
+    "s3fs>=2024.6.1",
     "scipy >=1.13",
     "tensorstore~=0.1.74",
     "typer ~=0.12.5",

webknossos/tests/administration/test_project.py

@@ -102,7 +102,7 @@ def test_project_get_owner() -> None:
 def test_project_get_tasks() -> None:
     """Test getting the task of a project."""
     teams = wk.Team.get_list()
-    ds_id = wk.Dataset.open_remote("l4_sample")._dataset_id
+    ds_id = wk.RemoteDataset.open("l4_sample")._dataset_id
 
     project = wk.Project.create(
         name="test_get_task_project",

webknossos/tests/administration/test_task.py

@@ -48,7 +48,7 @@ def test_get_task() -> None:
         needed_experience_domain="testing",
         needed_experience_value=0,
         starting_position=wk.Vec3Int(0, 0, 0),
-        dataset_id=wk.Dataset.open_remote("l4_sample")._dataset_id,
+        dataset_id=wk.RemoteDataset.open("l4_sample")._dataset_id,
     )
     task_by_id = wk.Task.get_by_id(tasks[0].task_id)
     assert task_by_id == tasks[0]
@@ -61,7 +61,7 @@ def test_task_get_project() -> None:
         needed_experience_domain="testing",
         needed_experience_value=0,
         starting_position=wk.Vec3Int(0, 0, 0),
-        dataset_id=wk.Dataset.open_remote("l4_sample")._dataset_id,
+        dataset_id=wk.RemoteDataset.open("l4_sample")._dataset_id,
     )
     project = tasks[0].get_project()
     assert project.name == TEST_PROJECT_NAME
@@ -82,7 +82,7 @@ def test_update_task() -> None:
         needed_experience_domain="testing",
         needed_experience_value=0,
         starting_position=wk.Vec3Int(0, 0, 0),
-        dataset_id=wk.Dataset.open_remote("l4_sample")._dataset_id,
+        dataset_id=wk.RemoteDataset.open("l4_sample")._dataset_id,
     )
     task = tasks[0]
     updated = task.update(
@@ -98,7 +98,7 @@ def test_delete_task() -> None:
         needed_experience_domain="testing",
         needed_experience_value=0,
         starting_position=wk.Vec3Int(0, 0, 0),
-        dataset_id=wk.Dataset.open_remote("l4_sample")._dataset_id,
+        dataset_id=wk.RemoteDataset.open("l4_sample")._dataset_id,
     )
     task = tasks[0]
     task.delete()
@@ -115,7 +115,7 @@ def test_task_list() -> None:
         needed_experience_domain="testing",
         needed_experience_value=0,
         starting_position=wk.Vec3Int(0, 0, 0),
-        dataset_id=wk.Dataset.open_remote("l4_sample")._dataset_id,
+        dataset_id=wk.RemoteDataset.open("l4_sample")._dataset_id,
     )
     task_list = wk.Task.get_list()
     assert len(task_list) > 0