1
1
import os
2
2
import pathlib
3
3
import warnings
4
+ import zipfile
4
5
from copy import deepcopy
5
- from typing import Any , Dict , Optional , Sequence , Tuple , Union
6
+ from typing import Dict , IO , Optional , Sequence , Tuple , Union
6
7
from zipfile import ZIP_DEFLATED , ZipFile
7
8
8
- from marshmallow import ValidationError , missing
9
+ from marshmallow import missing
9
10
10
11
from bioimageio import spec
11
12
from bioimageio .core .resource_io .nodes import ResourceDescription
13
+ from bioimageio .spec .io_ import resolve_rdf_source
12
14
from bioimageio .spec .shared import raw_nodes
13
- from bioimageio .spec .shared .common import get_class_name_from_type
15
+ from bioimageio .spec .shared .common import BIOIMAGEIO_CACHE_PATH , get_class_name_from_type
14
16
from bioimageio .spec .shared .raw_nodes import ResourceDescription as RawResourceDescription
15
17
from bioimageio .spec .shared .utils import PathToRemoteUriTransformer
16
18
from . import nodes
17
- from .common import BIOIMAGEIO_CACHE_PATH , yaml
18
- from .utils import _download_uri_to_local_path , resolve_local_uri , resolve_raw_resource_description , resolve_uri
19
+ from .utils import resolve_raw_resource_description , resolve_uri
19
20
20
-
21
- ROOT_PATH = "root_path"
22
21
serialize_raw_resource_description = spec .io_ .serialize_raw_resource_description
23
22
save_raw_resource_description = spec .io_ .save_raw_resource_description
24
23
25
24
25
+ def extract_resource_package (
26
+ source : Union [os .PathLike , IO , str , bytes , raw_nodes .URI ]
27
+ ) -> Tuple [dict , str , pathlib .Path ]:
28
+ """extract a zip source to BIOIMAGEIO_CACHE_PATH"""
29
+ source , source_name , root = resolve_rdf_source (source )
30
+ if isinstance (root , bytes ):
31
+ raise NotImplementedError ("package source was bytes" )
32
+
33
+ cache_folder = BIOIMAGEIO_CACHE_PATH / "extracted_packages"
34
+ cache_folder .mkdir (exist_ok = True , parents = True )
35
+
36
+ if isinstance (root , raw_nodes .URI ):
37
+ from urllib .request import urlretrieve
38
+
39
+ package_path = cache_folder / root .scheme / root .authority / root .path .strip ("/" ) / root .query
40
+ if (package_path / "rdf.yaml" ).exists ():
41
+ download = None
42
+ else :
43
+ download , header = urlretrieve (str (root ))
44
+
45
+ local_source = download
46
+ else :
47
+ download = None
48
+ local_source = root
49
+ package_path = cache_folder / root .relative_to (list (root .parents )[- 1 ])
50
+
51
+ if local_source is not None :
52
+ with zipfile .ZipFile (local_source ) as zf :
53
+ zf .extractall (package_path )
54
+
55
+ if not (package_path / "rdf.yaml" ).exists ():
56
+ raise FileNotFoundError (f"missing 'rdf.yaml' in { root } extracted from { download } " )
57
+
58
+ if download is not None :
59
+ try :
60
+ os .remove (download )
61
+ except Exception as e :
62
+ warnings .warn (f"Could not remove download { download } due to { e } " )
63
+
64
+ assert isinstance (package_path , pathlib .Path )
65
+ return source , source_name , package_path
66
+
67
+
26
68
def _replace_relative_paths_for_remote_source (
27
- raw_rd : RawResourceDescription , source : Union [Any , str , raw_nodes .URI ]
69
+ raw_rd : RawResourceDescription , root : Union [pathlib . Path , raw_nodes .URI , bytes ]
28
70
) -> RawResourceDescription :
29
- if isinstance (source , raw_nodes .URI ) or isinstance ( source , str ) and source . startswith ( "http" ):
71
+ if isinstance (root , raw_nodes .URI ):
30
72
# for a remote source relative paths are invalid; replace all relative file paths in source with URLs
31
- if isinstance (source , str ):
32
- source = raw_nodes .URI (source )
33
-
34
73
warnings .warn (
35
- f"changing file paths in RDF to URIs due to a remote { source .scheme } source "
74
+ f"changing file paths in RDF to URIs due to a remote { root .scheme } source "
36
75
"(may result in an invalid node)"
37
76
)
38
- raw_rd = PathToRemoteUriTransformer (remote_source = source ).transform (raw_rd )
39
- raw_rd .root_path = pathlib .Path () # root_path cannot be URI
77
+ raw_rd = PathToRemoteUriTransformer (remote_source = root ).transform (raw_rd )
78
+ root_path = pathlib .Path () # root_path cannot be URI
79
+ elif isinstance (root , pathlib .Path ):
80
+ if zipfile .is_zipfile (root ):
81
+ _ , _ , root_path = extract_resource_package (root )
82
+ else :
83
+ root_path = root
84
+ elif isinstance (root , bytes ):
85
+ raise NotImplementedError ("root as bytes (io)" )
86
+ else :
87
+ raise TypeError (root )
40
88
89
+ raw_rd .root_path = root_path
41
90
return raw_rd
42
91
43
92
44
93
def load_raw_resource_description (
45
- source : Union [os .PathLike , str , dict , raw_nodes .URI , RawResourceDescription ]
94
+ source : Union [dict , os .PathLike , IO , str , bytes , raw_nodes .URI ]
46
95
) -> RawResourceDescription :
47
96
"""load a raw python representation from a BioImage.IO resource description file (RDF).
48
97
Use `load_resource_description` for a more convenient representation.
@@ -53,12 +102,8 @@ def load_raw_resource_description(
53
102
Returns:
54
103
raw BioImage.IO resource
55
104
"""
56
- if isinstance (source , RawResourceDescription ):
57
- return source
58
-
59
- data , type_ = resolve_rdf_source_and_type (source )
60
- raw_rd = spec .load_raw_resource_description (data , update_to_current_format = True )
61
- raw_rd = _replace_relative_paths_for_remote_source (raw_rd , source )
105
+ raw_rd = spec .load_raw_resource_description (source , update_to_current_format = True )
106
+ raw_rd = _replace_relative_paths_for_remote_source (raw_rd , raw_rd .root_path )
62
107
return raw_rd
63
108
64
109
@@ -186,26 +231,6 @@ def _get_tmp_package_path(raw_rd: RawResourceDescription, weights_priority_order
186
231
return package_path
187
232
188
233
189
- def extract_resource_package (source : Union [os .PathLike , str , raw_nodes .URI ]) -> pathlib .Path :
190
- """extract a zip source to BIOIMAGEIO_CACHE_PATH"""
191
- local_source = resolve_uri (source )
192
- assert isinstance (local_source , pathlib .Path )
193
- cache_folder = BIOIMAGEIO_CACHE_PATH / "extracted_packages"
194
- cache_folder .mkdir (exist_ok = True , parents = True )
195
- package_path = cache_folder / f"{ local_source .stem } "
196
- with ZipFile (local_source ) as zf :
197
- zf .extractall (package_path )
198
-
199
- for rdf_name in ["rdf.yaml" , "model.yaml" , "rdf.yml" , "model.yml" ]:
200
- rdf_path = package_path / rdf_name
201
- if rdf_path .exists ():
202
- break
203
- else :
204
- raise FileNotFoundError (local_source / "rdf.yaml" )
205
-
206
- return rdf_path
207
-
208
-
209
234
def make_zip (
210
235
path : os .PathLike , content : Dict [str , Union [str , pathlib .Path ]], * , compression : int , compression_level : int
211
236
) -> None :
@@ -225,53 +250,3 @@ def make_zip(
225
250
myzip .writestr (arc_name , file_or_str_content )
226
251
else :
227
252
myzip .write (file_or_str_content , arcname = arc_name )
228
-
229
-
230
- def resolve_rdf_source_and_type (source : Union [os .PathLike , str , dict , raw_nodes .URI ]) -> Tuple [dict , str ]:
231
- if isinstance (source , dict ):
232
- data = source
233
- if ROOT_PATH not in data :
234
- data [ROOT_PATH ] = pathlib .Path ()
235
- else :
236
- data = get_dict_from_yaml_source (source )
237
-
238
- type_ = data .get ("type" , "model" ) # todo: remove default 'model' type
239
-
240
- return data , type_
241
-
242
-
243
- def get_dict_from_yaml_source (source : Union [os .PathLike , str , raw_nodes .URI , dict ]) -> dict :
244
- if isinstance (source , dict ):
245
- if ROOT_PATH not in source :
246
- source [ROOT_PATH ] = pathlib .Path ()
247
-
248
- return source
249
- elif isinstance (source , (str , os .PathLike , raw_nodes .URI )):
250
- source = resolve_local_uri (source , pathlib .Path ())
251
- else :
252
- raise TypeError (source )
253
-
254
- if isinstance (source , raw_nodes .URI ): # remote uri
255
- local_source = _download_uri_to_local_path (source )
256
- root_path = pathlib .Path ()
257
- else :
258
- local_source = source
259
- root_path = source .parent
260
-
261
- assert isinstance (local_source , pathlib .Path )
262
- if local_source .suffix == ".zip" :
263
- local_source = extract_resource_package (local_source )
264
- root_path = local_source .parent
265
-
266
- if local_source .suffix == ".yml" :
267
- warnings .warn (
268
- "suffix '.yml' is not recommended and will raise a ValidationError in the future. Use '.yaml' instead "
269
- "(https://yaml.org/faq.html)"
270
- )
271
- elif local_source .suffix != ".yaml" :
272
- raise ValidationError (f"invalid suffix { local_source .suffix } for source { source } " )
273
-
274
- data = yaml .load (local_source )
275
- assert isinstance (data , dict )
276
- data [ROOT_PATH ] = root_path
277
- return data
0 commit comments