1212)
1313import yaml
1414
15- from dcpy .models .connectors .esri import FeatureServer
15+ from dcpy .models .connectors .esri import FeatureServer , FeatureServerLayer
1616import dcpy .models .product .dataset .metadata as models
1717from dcpy .utils .logging import logger
1818
1919
20- def get_metadata (dataset : FeatureServer ) -> dict :
21- resp = requests .get (f"{ dataset .url } , params = {"f" : "pjson" })
20+ def get_feature_server_metadata (feature_server : FeatureServer ) -> dict :
21+ """Given a FeatureServer, return its metadata as a dictionary"""
22+ resp = requests .get (f"{ feature_server .url } , params = {"f" : "pjson" })
2223 resp .raise_for_status ()
2324 error = resp .json ().get ("error" ) # 200 responses might contain error details
2425 if error :
2526 raise Exception (f"Error fetching ESRI Server metadata: { error } )
2627 return resp .json ()
2728
2829
29- def get_data_last_updated (dataset : FeatureServer ) -> datetime :
30- metadata = get_metadata (dataset )
30+ def get_feature_server_layers (
31+ feature_server : FeatureServer ,
32+ ) -> list [FeatureServerLayer ]:
33+ """Given a FeatureServer, look up and return its available layers"""
34+ resp = get_feature_server_metadata (feature_server )
35+ return [
36+ FeatureServerLayer (
37+ server = feature_server .server ,
38+ name = feature_server .name ,
39+ layer_name = l ["name" ],
40+ layer_id = l ["id" ],
41+ )
42+ for l in resp ["layers" ]
43+ ]
44+
45+
46+ def resolve_layer (
47+ feature_server : FeatureServer ,
48+ layer_name : str | None = None ,
49+ layer_id : int | None = None ,
50+ ) -> FeatureServerLayer :
51+ """
52+ Given a FeatureServer, and optional layer name or id, resolve layer information
53+ There are a few different modes depending on what is provided
54+ For all modes, layers for the feature_server are looked up. Then, if
55+ - layer_name and layer_id provided - validate layer exists, return it
56+ - layer_name or layer_id provided - lookup layer by provided key
57+ - neither provided - if feature_server has single layer, return it. Otherwise, error
58+
59+ `assert` statements can hopefully be dropped - known bug in mypy to not correctly
60+ narrow types within tuples. See final comments in https://github.com/python/mypy/issues/12364
61+ """
62+ layers = get_feature_server_layers (feature_server )
63+ layer_labels = [l .layer_label for l in layers ]
64+
65+ match layer_id , layer_name :
66+ case None , None :
67+ if len (layers ) > 1 :
68+ raise ValueError (
69+ f"Feature server { feature_server } { layer_labels }
70+ )
71+ elif len (layers ) == 0 :
72+ raise LookupError (f"Feature server { feature_server } )
73+ else :
74+ return layers [0 ]
75+ case _, None :
76+ assert layer_id is not None
77+ layers_by_id = {l .layer_id : l for l in layers }
78+ if layer_id in layers_by_id :
79+ return layers_by_id [layer_id ]
80+ else :
81+ raise LookupError (
82+ f"Layer with id { layer_id } { feature_server } { layer_labels }
83+ )
84+ case None , _:
85+ assert layer_name is not None
86+ layers_by_name = {l .layer_name : l for l in layers }
87+ if layer_name in layers_by_name :
88+ return layers_by_name [layer_name ]
89+ else :
90+ raise LookupError (
91+ f"Layer with name '{ layer_name } { feature_server } { layer_labels }
92+ )
93+ case _:
94+ assert layer_name is not None
95+ assert layer_id is not None
96+ layer = FeatureServerLayer (
97+ server = feature_server .server ,
98+ name = feature_server .name ,
99+ layer_name = layer_name ,
100+ layer_id = layer_id ,
101+ )
102+ if layer not in layers :
103+ raise LookupError (
104+ f"Layer '{ layer } { feature_server }
105+ )
106+ return layer
107+
108+
109+ def get_layer_metadata (layer : FeatureServerLayer ) -> dict :
110+ """Given FeatureServerLayer, return its metadata as a dictionary"""
111+ resp = requests .get (f"{ layer .url } , params = {"f" : "pjson" })
112+ resp .raise_for_status ()
113+ error = resp .json ().get ("error" ) # 200 responses might contain error details
114+ if error :
115+ raise Exception (f"Error fetching ESRI Server metadata: { error } )
116+ return resp .json ()
117+
118+
119+ def get_data_last_updated (layer : FeatureServerLayer ) -> datetime :
120+ """Given FeatureServerLayer, lookup date of last data edit"""
121+ metadata = get_layer_metadata (layer )
31122 ## returned timestamp has milliseconds, fromtimestamp expects seconds
32123 return datetime .fromtimestamp (metadata ["editingInfo" ]["dataLastEditDate" ] / 1e3 )
33124
34125
35- def query_dataset (dataset : FeatureServer , params : dict ) -> dict :
36- resp = requests .post (f"{ dataset .url } , data = params )
126+ def query_layer (layer : FeatureServerLayer , params : dict ) -> dict :
127+ """
128+ Wrapper to query data for a FeatureServerLayer.
129+ Arguments are `layer`, a FeatureServerLayer, and `params`, which are kwargs for the api call
130+
131+ For these params, we commonly use
132+ - where: essentially a sql where clause. Default of "1=1" should be provided
133+ - outFields: fields to select. Required when querying data. Default of "*" should be provided
134+ - outSr: spatial reference system to get data in
135+ - f: output format. Default of "geojson" should be provided
136+ - returnIdsOnly: boolean flag to only return ids. Doesn't have same query limits as data queries
137+ - objectIds: list of object ids (that can be queried separately) to return. Useful in
138+
139+ Exhaustive list of possible params are here:
140+ https://developers.arcgis.com/rest/services-reference/enterprise/query-feature-service-layer/#request-parameters
141+ """
142+ resp = requests .post (f"{ layer .url } , data = params )
37143 resp .raise_for_status ()
38144 return resp .json ()
39145
40146
41- def get_dataset (dataset : FeatureServer , crs : int ) -> dict :
42- CHUNK_SIZE = 100
147+ def get_layer (layer : FeatureServerLayer , crs : int , chunk_size = 100 ) -> dict :
148+ """
149+ Given FeatureServerLayer and desired crs, fetches entire layer as geojson (dict)
150+ """
43151 params = {"where" : "1=1" , "outFields" : "*" , "outSr" : crs , "f" : "geojson" }
44152
45153 # there is a limit of 2000 features on the server, unless we limit to objectIds only
46- # so first, we get ids, then we chunk to get full dataset
154+ # so first, we get ids, then we chunk to get full layer
47155 obj_params = params .copy ()
48156 obj_params ["returnIdsOnly" ] = True
49- object_id_resp = query_dataset ( dataset , obj_params )
157+ object_id_resp = query_layer ( layer , obj_params )
50158 object_ids = cast (list [int ], object_id_resp ["properties" ]["objectIds" ])
51159
52160 features = []
@@ -60,17 +168,17 @@ def get_dataset(dataset: FeatureServer, crs: int) -> dict:
60168 transient = True ,
61169 ) as progress :
62170 task = progress .add_task (
63- f"[green]Downloading [bold]{ dataset .name } , total = len (object_ids )
171+ f"[green]Downloading [bold]{ layer .name } , total = len (object_ids )
64172 )
65173
66174 def _downcase_properties_keys (feat ):
67175 feat ["properties" ] = {k .lower (): v for k , v in feat ["properties" ].items ()}
68176 return feat
69177
70- for i in range (0 , len (object_ids ), CHUNK_SIZE ):
71- params ["objectIds" ] = object_ids [i : i + CHUNK_SIZE ]
72- chunk = query_dataset ( dataset , params )
73- progress .update (task , completed = i + CHUNK_SIZE )
178+ for i in range (0 , len (object_ids ), chunk_size ):
179+ params ["objectIds" ] = object_ids [i : i + chunk_size ]
180+ chunk = query_layer ( layer , params )
181+ progress .update (task , completed = i + chunk_size )
74182 features += [_downcase_properties_keys (feat ) for feat in chunk ["features" ]]
75183
76184 return {"type" : "FeatureCollection" , "crs" : crs , "features" : features }
0 commit comments