Zip Store

.github/workflows/ci.yml

@@ -14,6 +14,14 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
       fail-fast: false
     runs-on: ${{ matrix.os }}
+    services:
+      s3mock:
+        # This service will only start if the runner is Linux
+        image: ${{ matrix.os == 'ubuntu-latest' && 'adobe/s3mock:3.11.0' || '' }}
+        ports:
+          - 9090:9090
+        env:
+          initialBuckets: zarr-test-bucket
     defaults:
       run:
         shell: bash
@@ -49,8 +57,13 @@ jobs:
       - name: Test
         env:
           MAVEN_OPTS: "-Xmx6g"
-        run: mvn --no-transfer-progress test -DargLine="-Xmx6g"
-
+        run: |
+          if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+            mvn --no-transfer-progress test -DargLine="-Xmx6g"
+          else
+            # Skip S3 tests on Windows/macOS where the service isn't running (labeled with @Tag("s3"))
+            mvn --no-transfer-progress test -DargLine="-Xmx6g" -DexcludedGroups="s3"
+          fi
       - name: Assemble JAR
         run: mvn package -DskipTests
 

.gitignore

@@ -45,4 +45,5 @@ build/
 /main.py
 /pyproject.toml
 /uv.lock
-**/__pycache__
+**/__pycache__
+/dependency-reduced-pom.xml

pom.xml

@@ -121,6 +121,11 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.28.0</version>
+        </dependency>
     </dependencies>
 
     <repositories>
@@ -139,6 +144,10 @@
                 <version>3.2.5</version>
                 <configuration>
                     <useSystemClassLoader>false</useSystemClassLoader>
+                    <environmentVariables>
+                        <!-- Force Testcontainers to use a newer Docker API version supported by your local Daemon -->
+                        <DOCKER_API_VERSION>1.44</DOCKER_API_VERSION>
+                    </environmentVariables>
                 </configuration>
             </plugin>
             <plugin>

src/main/java/dev/zarr/zarrjava/core/Array.java

@@ -3,6 +3,7 @@
 import dev.zarr.zarrjava.ZarrException;
 import dev.zarr.zarrjava.core.codec.CodecPipeline;
 import dev.zarr.zarrjava.store.FilesystemStore;
+import dev.zarr.zarrjava.store.Store;
 import dev.zarr.zarrjava.store.StoreHandle;
 import dev.zarr.zarrjava.utils.IndexingUtils;
 import dev.zarr.zarrjava.utils.MultiArrayUtils;
@@ -16,6 +17,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public abstract class Array extends AbstractNode {
@@ -289,6 +293,15 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
         if (parallel) {
             chunkStream = chunkStream.parallel();
         }
+
+        boolean isListableStore = storeHandle.store instanceof Store.ListableStore;
+        Set<List<String>> existingKeys;
+        if (isListableStore) {
+            existingKeys = storeHandle.list().map(Arrays::asList).collect(Collectors.toSet());
+        } else {
+            existingKeys = null;
+        }
+
         chunkStream.forEach(
                 chunkCoords -> {
                     try {
@@ -306,9 +319,13 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
 
                         final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
                         final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
-                        if (!chunkHandle.exists()) {
-                            return;
-                        }
+
+                        // chunkHandle.exists() can be expensive on some store types, so we optimize for ListableStore
+                        if (isListableStore) {
+                            if (existingKeys.stream().noneMatch(Arrays.asList(chunkKeys)::equals))
+                                return;
+                        } else if (!chunkHandle.exists()) return;
+
                         if (codecPipeline.supportsPartialDecode()) {
                             final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
                                     Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);

src/main/java/dev/zarr/zarrjava/core/Group.java

@@ -9,7 +9,6 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Objects;
 import java.util.stream.Stream;
 
 public abstract class Group extends AbstractNode {
@@ -64,20 +63,15 @@ public static Group open(String path) throws IOException, ZarrException {
     }
 
     @Nullable
-    public abstract Node get(String key) throws ZarrException, IOException;
+    public abstract Node get(String[] key) throws ZarrException, IOException;
 
-    public Stream<Node> list() {
-        return storeHandle.list()
-                .map(key -> {
-                    try {
-                        return get(key);
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
-                })
-                .filter(Objects::nonNull);
+    @Nullable
+    public Node get(String key) throws ZarrException, IOException {
+        return get(new String[]{key});
     }
 
+    public abstract Stream<Node> list();
+
     public Node[] listAsArray() {
         try (Stream<Node> nodeStream = list()) {
             return nodeStream.toArray(Node[]::new);