Merge remote-tracking branch 'origin/string-arguments-for-codecs' into string-arguments-for-codecs

Author: brokkoli71

changes/2862.bugfix.rst

@@ -3,6 +3,42 @@ Release notes
 
 .. towncrier release notes start
 
+3.0.8 (2025-05-19)
+------------------
+
+.. warning::
+
+    In versions 3.0.0 to 3.0.7 opening arrays or groups with ``mode='a'`` (the default for many builtin functions)
+    would cause any existing paths in the store to be deleted. This is fixed in 3.0.8, and
+    we recommend all users upgrade to avoid this bug that could cause unintentional data loss.
+
+Features
+~~~~~~~~
+
+- Added a `print_debug_info` function for bug reports. (:issue:`2913`)
+
+
+Bugfixes
+~~~~~~~~
+
+- Fix a bug that prevented the number of initialized chunks being counted properly. (:issue:`2862`)
+- Fixed sharding with GPU buffers. (:issue:`2978`)
+- Fix structured `dtype` fill value serialization for consolidated metadata (:issue:`2998`)
+- It is now possible to specify no compressor when creating a zarr format 2 array.
+  This can be done by passing ``compressor=None`` to the various array creation routines.
+
+  The default behaviour of automatically choosing a suitable default compressor remains if the compressor argument is not given.
+  To reproduce the behaviour in previous zarr-python versions when ``compressor=None`` was passed, pass ``compressor='auto'`` instead. (:issue:`3039`)
+- Fixed the typing of ``dimension_names`` arguments throughout so that it now accepts iterables that contain `None` alongside `str`. (:issue:`3045`)
+- Using various functions to open data with ``mode='a'`` no longer deletes existing data in the store. (:issue:`3062`)
+
+
+Misc
+~~~~
+
+- :issue:`2972`, :issue:`3027`, :issue:`3049`
+
+
 3.0.7 (2025-04-22)
 ------------------
 

changes/2972.misc.rst

@@ -37,6 +37,54 @@
 # in case setuptools scm screw up and find version to be 0.0.0
 assert not __version__.startswith("0.0.0")
 
+
+def print_debug_info() -> None:
+    """
+    Print version info for use in bug reports.
+    """
+    import platform
+    from importlib.metadata import version
+
+    def print_packages(packages: list[str]) -> None:
+        not_installed = []
+        for package in packages:
+            try:
+                print(f"{package}: {version(package)}")
+            except ModuleNotFoundError:
+                not_installed.append(package)
+        if not_installed:
+            print("\n**Not Installed:**")
+            for package in not_installed:
+                print(package)
+
+    required = [
+        "packaging",
+        "numpy",
+        "numcodecs",
+        "typing_extensions",
+        "donfig",
+    ]
+    optional = [
+        "botocore",
+        "cupy-cuda12x",
+        "fsspec",
+        "numcodecs",
+        "s3fs",
+        "gcsfs",
+        "universal-pathlib",
+        "rich",
+        "obstore",
+    ]
+
+    print(f"platform: {platform.platform()}")
+    print(f"python: {platform.python_version()}")
+    print(f"zarr: {__version__}\n")
+    print("**Required dependencies:**")
+    print_packages(required)
+    print("\n**Optional dependencies:**")
+    print_packages(optional)
+
+
 __all__ = [
     "Array",
     "AsyncArray",
@@ -67,6 +115,7 @@
     "open_consolidated",
     "open_group",
     "open_like",
+    "print_debug_info",
     "save",
     "save_array",
     "save_group",

changes/2978.bugfix.rst

@@ -88,7 +88,7 @@
 
 _READ_MODES: tuple[AccessModeLiteral, ...] = ("r", "r+", "a")
 _CREATE_MODES: tuple[AccessModeLiteral, ...] = ("a", "w", "w-")
-_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("a", "r+", "w")
+_OVERWRITE_MODES: tuple[AccessModeLiteral, ...] = ("w",)
 
 
 def _infer_overwrite(mode: AccessModeLiteral) -> bool:
@@ -817,7 +817,6 @@ async def open_group(
         warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2)
 
     store_path = await make_store_path(store, mode=mode, storage_options=storage_options, path=path)
-
     if attributes is None:
         attributes = {}
 

changes/2998.bugfix.md

@@ -858,6 +858,7 @@ def create_array(
         Ignored otherwise.
     overwrite : bool, default False
         Whether to overwrite an array with the same name in the store, if one exists.
+        If `True`, all existing paths in the store will be deleted.
     config : ArrayConfigLike, optional
         Runtime configuration for the array.
     write_data : bool

changes/3027.misc.rst

@@ -17,6 +17,7 @@
 import zarr
 from zarr import Array
 from zarr.abc.store import Store
+from zarr.codecs.bytes import BytesCodec
 from zarr.core.buffer import Buffer, BufferPrototype, cpu, default_buffer_prototype
 from zarr.core.sync import SyncMixin
 from zarr.storage import LocalStore, MemoryStore
@@ -108,7 +109,15 @@ def add_array(
         assume(self.can_add(path))
         note(f"Adding array:  path='{path}'  shape={array.shape}  chunks={chunks}")
         for store in [self.store, self.model]:
-            zarr.array(array, chunks=chunks, path=path, store=store, fill_value=fill_value)
+            zarr.array(
+                array,
+                chunks=chunks,
+                path=path,
+                store=store,
+                fill_value=fill_value,
+                # Chose bytes codec to avoid wasting time compressing the data being written
+                codecs=[BytesCodec()],
+            )
         self.all_arrays.add(path)
 
     # @precondition(lambda self: bool(self.all_groups))

changes/3039.bugfix.rst

@@ -4,13 +4,15 @@
 from typing import TYPE_CHECKING
 
 import zarr.codecs
+import zarr.storage
 
 if TYPE_CHECKING:
     import pathlib
 
     from zarr.abc.store import Store
     from zarr.core.common import JSON, MemoryOrder, ZarrFormat
 
+import contextlib
 import warnings
 from typing import Literal
 
@@ -27,9 +29,9 @@
     create,
     create_array,
     create_group,
+    from_array,
     group,
     load,
-    open,
     open_group,
     save,
     save_array,
@@ -41,6 +43,10 @@
 from zarr.storage._utils import normalize_path
 from zarr.testing.utils import gpu_test
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
 
 def test_create(memory_store: Store) -> None:
     store = memory_store
@@ -135,28 +141,28 @@ async def test_open_array(memory_store: MemoryStore, zarr_format: ZarrFormat) ->
     store = memory_store
 
     # open array, create if doesn't exist
-    z = open(store=store, shape=100, zarr_format=zarr_format)
+    z = zarr.api.synchronous.open(store=store, shape=100, zarr_format=zarr_format)
     assert isinstance(z, Array)
     assert z.shape == (100,)
 
     # open array, overwrite
     # store._store_dict = {}
     store = MemoryStore()
-    z = open(store=store, shape=200, zarr_format=zarr_format)
+    z = zarr.api.synchronous.open(store=store, shape=200, zarr_format=zarr_format)
     assert isinstance(z, Array)
     assert z.shape == (200,)
 
     # open array, read-only
     store_cls = type(store)
     ro_store = await store_cls.open(store_dict=store._store_dict, read_only=True)
-    z = open(store=ro_store, mode="r")
+    z = zarr.api.synchronous.open(store=ro_store, mode="r")
     assert isinstance(z, Array)
     assert z.shape == (200,)
     assert z.read_only
 
     # path not found
     with pytest.raises(FileNotFoundError):
-        open(store="doesnotexist", mode="r", zarr_format=zarr_format)
+        zarr.api.synchronous.open(store="doesnotexist", mode="r", zarr_format=zarr_format)
 
 
 @pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True)
@@ -233,12 +239,12 @@ def test_save(store: Store, n_args: int, n_kwargs: int) -> None:
             save(store)
     elif n_args == 1 and n_kwargs == 0:
         save(store, *args)
-        array = open(store)
+        array = zarr.api.synchronous.open(store)
         assert isinstance(array, Array)
         assert_array_equal(array[:], data)
     else:
         save(store, *args, **kwargs)  # type: ignore [arg-type]
-        group = open(store)
+        group = zarr.api.synchronous.open(store)
         assert isinstance(group, Group)
         for array in group.array_values():
             assert_array_equal(array[:], data)
@@ -1077,7 +1083,7 @@ def test_tree() -> None:
 def test_open_positional_args_deprecated() -> None:
     store = MemoryStore()
     with pytest.warns(FutureWarning, match="pass"):
-        open(store, "w", shape=(1,))
+        zarr.api.synchronous.open(store, "w", shape=(1,))
 
 
 def test_save_array_positional_args_deprecated() -> None:
@@ -1236,3 +1242,62 @@ def test_v2_with_v3_compressor() -> None:
         zarr.create(
             store={}, shape=(1), dtype="uint8", zarr_format=2, compressor=zarr.codecs.BloscCodec()
         )
+
+
+def add_empty_file(path: Path) -> Path:
+    fpath = path / "a.txt"
+    fpath.touch()
+    return fpath
+
+
+@pytest.mark.parametrize("create_function", [create_array, from_array])
+@pytest.mark.parametrize("overwrite", [True, False])
+def test_no_overwrite_array(tmp_path: Path, create_function: Callable, overwrite: bool) -> None:  # type:ignore[type-arg]
+    store = zarr.storage.LocalStore(tmp_path)
+    existing_fpath = add_empty_file(tmp_path)
+
+    assert existing_fpath.exists()
+    create_function(store=store, data=np.ones(shape=(1,)), overwrite=overwrite)
+    if overwrite:
+        assert not existing_fpath.exists()
+    else:
+        assert existing_fpath.exists()
+
+
+@pytest.mark.parametrize("create_function", [create_group, group])
+@pytest.mark.parametrize("overwrite", [True, False])
+def test_no_overwrite_group(tmp_path: Path, create_function: Callable, overwrite: bool) -> None:  # type:ignore[type-arg]
+    store = zarr.storage.LocalStore(tmp_path)
+    existing_fpath = add_empty_file(tmp_path)
+
+    assert existing_fpath.exists()
+    create_function(store=store, overwrite=overwrite)
+    if overwrite:
+        assert not existing_fpath.exists()
+    else:
+        assert existing_fpath.exists()
+
+
+@pytest.mark.parametrize("open_func", [zarr.open, open_group])
+@pytest.mark.parametrize("mode", ["r", "r+", "a", "w", "w-"])
+def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> None:  # type:ignore[type-arg]
+    store = zarr.storage.LocalStore(tmp_path)
+    existing_fpath = add_empty_file(tmp_path)
+
+    assert existing_fpath.exists()
+    with contextlib.suppress(FileExistsError, FileNotFoundError):
+        open_func(store=store, mode=mode)
+    if mode == "w":
+        assert not existing_fpath.exists()
+    else:
+        assert existing_fpath.exists()
+
+
+def test_no_overwrite_load(tmp_path: Path) -> None:
+    store = zarr.storage.LocalStore(tmp_path)
+    existing_fpath = add_empty_file(tmp_path)
+
+    assert existing_fpath.exists()
+    with contextlib.suppress(NotImplementedError):
+        zarr.load(store)
+    assert existing_fpath.exists()

changes/3045.bugfix.rst

@@ -1,3 +1,5 @@
+import pytest
+
 import zarr
 
 
@@ -9,3 +11,19 @@ def test_exports() -> None:
 
     for export in __all__:
         getattr(zarr, export)
+
+
+def test_print_debug_info(capsys: pytest.CaptureFixture[str]) -> None:
+    """
+    Ensure that print_debug_info does not raise an error
+    """
+    from importlib.metadata import version
+
+    from zarr import __version__, print_debug_info
+
+    print_debug_info()
+    captured = capsys.readouterr()
+    # test that at least some of what we expect is
+    # printed out
+    assert f"zarr: {__version__}" in captured.out
+    assert f"numpy: {version('numpy')}" in captured.out