Merge pull request #130 from bioimage-io/add_test_resource

Add test_resource

bioimageio/core/__main__.py

@@ -1,3 +1,4 @@
+import enum
 import json
 import os
 from glob import glob
@@ -7,8 +8,14 @@
 
 import typer
 
-from bioimageio.core import __version__, prediction, commands
+from bioimageio.core import __version__, prediction, commands, resource_tests
 from bioimageio.spec.__main__ import app
+from bioimageio.spec.model.raw_nodes import WeightsFormat
+
+try:
+    from typing import get_args
+except ImportError:
+    from typing_extensions import get_args  # type: ignore
 
 try:
     from bioimageio.core.weight_converter import torch as torch_converter
@@ -40,28 +47,57 @@ def package(
 
 # if we want to use something like "choice" for the weight formats, we need to use an enum, see:
 # https://github.com/tiangolo/typer/issues/182
+WeightFormatEnum = enum.Enum("WeightFormatEnum", get_args(WeightsFormat))
+
+
 @app.command()
 def test_model(
     model_rdf: str = typer.Argument(
         ..., help="Path or URL to the model resource description file (rdf.yaml) or zipped model."
     ),
-    weight_format: Optional[str] = typer.Argument(None, help="The weight format to use."),
+    weight_format: Optional[WeightFormatEnum] = typer.Argument(None, help="The weight format to use."),
     devices: Optional[List[str]] = typer.Argument(None, help="Devices for running the model."),
     decimal: int = typer.Argument(4, help="The test precision."),
 ) -> int:
     # this is a weird typer bug: default devices are empty tuple although they should be None
     if len(devices) == 0:
         devices = None
-    test_passed = prediction.test_model(model_rdf, weight_format=weight_format, devices=devices, decimal=decimal)
-    if test_passed:
+    summary = resource_tests.test_model(model_rdf, weight_format=weight_format, devices=devices, decimal=decimal)
+    if summary["error"] is None:
         print(f"Model test for {model_rdf} has passed.")
+        return 0
     else:
         print(f"Model test for {model_rdf} has FAILED!")
-    ret_code = 0 if test_passed else 1
-    return ret_code
+        print(summary)
+        return 1
+
+
+test_model.__doc__ = resource_tests.test_model.__doc__
+
+
+@app.command()
+def test_resource(
+    rdf: str = typer.Argument(
+        ..., help="Path or URL to the resource description file (rdf.yaml) or zipped resource package."
+    ),
+    weight_format: Optional[WeightFormatEnum] = typer.Argument(None, help="(for model only) The weight format to use."),
+    devices: Optional[List[str]] = typer.Argument(None, help="(for model only) Devices for running the model."),
+    decimal: int = typer.Argument(4, help="(for model only) The test precision."),
+) -> int:
+    # this is a weird typer bug: default devices are empty tuple although they should be None
+    if len(devices) == 0:
+        devices = None
+    summary = resource_tests.test_resource(rdf, weight_format=weight_format, devices=devices, decimal=decimal)
+    if summary["error"] is None:
+        print(f"Resource test for {rdf} has passed.")
+        return 0
+    else:
+        print(f"Resource test for {rdf} has FAILED!")
+        print(summary)
+        return 1
 
 
-test_model.__doc__ = prediction.test_model.__doc__
+test_resource.__doc__ = resource_tests.test_resource.__doc__
 
 
 @app.command()

bioimageio/core/prediction.py

@@ -1,6 +1,5 @@
 import collections
 import os
-import warnings
 from copy import deepcopy
 from itertools import product
 from pathlib import Path
@@ -9,19 +8,16 @@
 import imageio
 import numpy as np
 import xarray as xr
+from tqdm import tqdm
 
 from bioimageio.core import load_resource_description
-from bioimageio.core.resource_io.nodes import InputTensor, Model, OutputTensor
 from bioimageio.core.prediction_pipeline import PredictionPipeline, create_prediction_pipeline
-from tqdm import tqdm
+from bioimageio.core.resource_io.nodes import ImplicitOutputShape, InputTensor, Model, OutputTensor
 
 
 #
 # utility functions for prediction
 #
-from bioimageio.core.resource_io.nodes import ImplicitOutputShape, URI
-
-
 def require_axes(im, axes):
     is_volume = "z" in axes
     # we assume images / volumes are loaded as one of
@@ -474,32 +470,3 @@ def predict_images(
             outp = [outp]
 
         _predict_sample(prediction_pipeline, inp, outp, padding, tiling)
-
-
-def test_model(model_rdf: Union[URI, Path, str], weight_format=None, devices=None, decimal=4):
-    """Test whether the test output(s) of a model can be reproduced.
-
-    Returns True if the test passes, otherwise returns False and issues a warning.
-    """
-    model = load_resource_description(model_rdf)
-    assert isinstance(model, Model)
-    prediction_pipeline = create_prediction_pipeline(
-        bioimageio_model=model, devices=devices, weight_format=weight_format
-    )
-    inputs = [np.load(str(in_path)) for in_path in model.test_inputs]
-    results = predict(prediction_pipeline, inputs)
-    if isinstance(results, (np.ndarray, xr.DataArray)):
-        results = [results]
-
-    expected = [np.load(str(out_path)) for out_path in model.test_outputs]
-    if len(results) != len(expected):
-        warnings.warn(f"Number of outputs and number of expected outputs disagree: {len(results)} != {len(expected)}")
-        return False
-
-    for res, exp in zip(results, expected):
-        try:
-            np.testing.assert_array_almost_equal(res, exp, decimal=decimal)
-        except AssertionError as e:
-            warnings.warn(f"Output and expected output disagree:\n {e}")
-            return False
-    return True

bioimageio/core/prediction_pipeline/_model_adapters/_model_adapter.py

@@ -58,7 +58,9 @@ def create_model_adapter(
             adapter_cls = _get_model_adapter(weight)
             return adapter_cls(bioimageio_model=bioimageio_model, devices=devices)
 
-    raise NotImplementedError(f"No supported weight_formats in {spec.weights.keys()}")
+    raise RuntimeError(
+        f"weight format {weight_format} not among weight formats listed in model: {list(spec.weights.keys())}"
+    )
 
 
 def _get_model_adapter(weight_format: str) -> Type[ModelAdapter]:

bioimageio/core/resource_io/io_.py

@@ -40,7 +40,10 @@ def extract_resource_package(
         if (package_path / "rdf.yaml").exists():
             download = None
         else:
-            download, header = urlretrieve(str(root))
+            try:
+                download, header = urlretrieve(str(root))
+            except Exception as e:
+                raise RuntimeError(f"Failed to download {str(root)} ({e})")
 
         local_source = download
     else:
@@ -91,7 +94,7 @@ def _replace_relative_paths_for_remote_source(
 
 
 def load_raw_resource_description(
-    source: Union[dict, os.PathLike, IO, str, bytes, raw_nodes.URI]
+    source: Union[dict, os.PathLike, IO, str, bytes, raw_nodes.URI, RawResourceDescription]
 ) -> RawResourceDescription:
     """load a raw python representation from a BioImage.IO resource description file (RDF).
     Use `load_resource_description` for a more convenient representation.
@@ -102,13 +105,16 @@ def load_raw_resource_description(
     Returns:
         raw BioImage.IO resource
     """
+    if isinstance(source, RawResourceDescription):
+        return source
+
     raw_rd = spec.load_raw_resource_description(source, update_to_current_format=True)
     raw_rd = _replace_relative_paths_for_remote_source(raw_rd, raw_rd.root_path)
     return raw_rd
 
 
 def load_resource_description(
-    source: Union[RawResourceDescription, os.PathLike, str, dict, raw_nodes.URI],
+    source: Union[RawResourceDescription, ResourceDescription, os.PathLike, str, dict, raw_nodes.URI],
     *,
     weights_priority_order: Optional[Sequence[str]] = None,  # model only
 ) -> ResourceDescription:
@@ -123,6 +129,9 @@ def load_resource_description(
         BioImage.IO resource
     """
     source = deepcopy(source)
+    if isinstance(source, ResourceDescription):
+        return source
+
     raw_rd = load_raw_resource_description(source)
 
     if weights_priority_order is not None:

bioimageio/core/resource_io/utils.py

@@ -276,9 +276,8 @@ def _download_uri_to_local_path(uri: raw_nodes.URI) -> pathlib.Path:
         local_path.parent.mkdir(parents=True, exist_ok=True)
         try:
             urlretrieve(str(uri), str(local_path))
-        except Exception:
-            logging.getLogger("download").error("Failed to download %s", uri)
-            raise
+        except Exception as e:
+            raise RuntimeError(f"Failed to download {uri} ({e})")
 
     return local_path
 

bioimageio/core/resource_tests.py

@@ -0,0 +1,81 @@
+import traceback
+import warnings
+from pathlib import Path
+from typing import List, Optional, Union
+
+import numpy as np
+import xarray as xr
+
+from bioimageio.core import load_resource_description
+from bioimageio.core.prediction import predict
+from bioimageio.core.prediction_pipeline import create_prediction_pipeline
+from bioimageio.core.resource_io.nodes import Model, ResourceDescription, URI
+from bioimageio.spec.model.raw_nodes import WeightsFormat
+from bioimageio.spec.shared.raw_nodes import ResourceDescription as RawResourceDescription
+
+
+def test_model(
+    model_rdf: Union[URI, Path, str],
+    weight_format: Optional[WeightsFormat] = None,
+    devices: Optional[List[str]] = None,
+    decimal: int = 4,
+) -> dict:
+    """Test whether the test output(s) of a model can be reproduced.
+
+    Returns summary dict with "error" and "traceback" key; summary["error"] is None if no errors were encountered.
+    """
+    model = load_resource_description(model_rdf)
+    if isinstance(model, Model):
+        return test_resource(model, weight_format=weight_format, devices=devices, decimal=decimal)
+    else:
+        return {"error": f"Expected RDF type Model, got {type(model)} instead.", "traceback": None}
+
+
+def test_resource(
+    model_rdf: Union[RawResourceDescription, ResourceDescription, URI, Path, str],
+    *,
+    weight_format: Optional[WeightsFormat] = None,
+    devices: Optional[List[str]] = None,
+    decimal: int = 4,
+):
+    """Test RDF dynamically
+
+    Returns summary dict with "error" and "traceback" key; summary["error"] is None if no errors were encountered.
+    """
+    error: Optional[str] = None
+    tb: Optional = None
+
+    try:
+        model = load_resource_description(model_rdf)
+    except Exception as e:
+        error = str(e)
+        tb = traceback.format_tb(e.__traceback__)
+    else:
+        if isinstance(model, Model):
+            try:
+                prediction_pipeline = create_prediction_pipeline(
+                    bioimageio_model=model, devices=devices, weight_format=weight_format
+                )
+                inputs = [np.load(str(in_path)) for in_path in model.test_inputs]
+                results = predict(prediction_pipeline, inputs)
+                if isinstance(results, (np.ndarray, xr.DataArray)):
+                    results = [results]
+
+                expected = [np.load(str(out_path)) for out_path in model.test_outputs]
+                if len(results) != len(expected):
+                    error = (
+                        f"Number of outputs and number of expected outputs disagree: {len(results)} != {len(expected)}"
+                    )
+                else:
+                    for res, exp in zip(results, expected):
+                        try:
+                            np.testing.assert_array_almost_equal(res, exp, decimal=decimal)
+                        except AssertionError as e:
+                            error = f"Output and expected output disagree:\n {e}"
+            except Exception as e:
+                error = str(e)
+                tb = traceback.format_tb(e.__traceback__)
+
+    # todo: add tests for non-model resources
+
+    return {"error": error, "traceback": tb}

tests/test_prediction.py

@@ -9,11 +9,17 @@
 
 
 def test_test_model(unet2d_nuclei_broad_model):
-    from bioimageio.core.prediction import test_model
+    from bioimageio.core.resource_tests import test_model
 
     assert test_model(unet2d_nuclei_broad_model)
 
 
+def test_test_resource(unet2d_nuclei_broad_model):
+    from bioimageio.core.resource_tests import test_resource
+
+    assert test_resource(unet2d_nuclei_broad_model)
+
+
 def test_predict_image(unet2d_fixed_shape_or_not, tmpdir):
     any_model = unet2d_fixed_shape_or_not  # todo: replace 'unet2d_fixed_shape_or_not' with 'any_model'
     from bioimageio.core.prediction import predict_image