Clip rock and most water data by pack GeoJSON boundary

Did the opposite for waterways and included them all, because it's often
useful to see where water flows beyond the pack boundary. Also got more
specific with CA TIGER water sources so we don't load the entire state when
making a pack for a single region.

create_pack.py

@@ -56,6 +56,7 @@ def generate_geojson(sources, args, data):
     if os.path.isfile(outpath):
         log(f"{outpath} exists, skipping GeoJSON generation...")
         return outpath
+    generate_sources(sources, args)
     pack_hulls = []
     for source in sources:
         source_output_path = f"sources/work-{source}/units.geojson"
@@ -235,7 +236,6 @@ def find_water_sources(geojson_path):
     }
     if not data["id"] or len(data["id"]) == 0:
         raise ValueError("You must specify an ID")
-    generate_sources(data["rock"], args)
     geojson_path = generate_geojson(data["rock"], args, data)
     data["geojson"] = {
         "$ref": f"file://./{os.path.basename(geojson_path)}"

packs.py

@@ -180,6 +180,7 @@ def make_rocks_for_pack(pack_id, clean=False, procs=2):
     make_rocks(
         pack["rock"],
         bbox=pack["bbox"],
+        geojson_path=pack["geojson_path"],
         clean=clean,
         path=rocks_mbtiles_path,
         procs=procs)
@@ -227,9 +228,7 @@ def make_water_for_pack(pack_id, clean=False, procs=2):
     make_water(
         pack["water"],
         bbox=pack["bbox"],
-        # TODO make pack options to clip water / rocks / ways by the bbox
-        # or not. sometimes it makes more sense to include everythign in
-        # the sources
+        geojson_path=pack["geojson_path"],
         clean=clean,
         path=water_mbtiles_path,
         procs=procs)

packs/us-ca-carrizo.json

@@ -21,6 +21,8 @@
     "water": [
         "nhdplus_h_1803_hu4",
         "nhdplus_h_1806_hu4",
-        "tiger_ca_water"
+        "tiger_water_06029",
+        "tiger_water_06079",
+        "tiger_water_06083"
     ]
 }

packs/us-ca-joshua.json

@@ -20,6 +20,7 @@
     ],
     "water": [
         "nhdplus_h_1810_hu4",
-        "tiger_ca_water"
+        "tiger_water_06065",
+        "tiger_water_06071"
     ]
 }

packs/us-ca-la.json

@@ -22,6 +22,11 @@
         "nhdplus_h_1807_hu4",
         "nhdplus_h_1809_hu4",
         "nhdplus_h_1810_hu4",
-        "tiger_ca_water"
+        "tiger_water_06059",
+        "tiger_water_06065",
+        "tiger_water_06111",
+        "tiger_water_06037",
+        "tiger_water_06073",
+        "tiger_water_06071"
     ]
 }

packs/us-ca-mojave.json

@@ -26,6 +26,7 @@
         "nhdplus_h_1606_hu4",
         "nhdplus_h_1809_hu4",
         "nhdplus_h_1810_hu4",
-        "tiger_ca_water"
+        "tiger_water_32003",
+        "tiger_water_06071"
     ]
 }

packs/us-ca-oakland.json

@@ -19,6 +19,7 @@
     ],
     "water": [
         "nhdplus_h_1805_hu4",
-        "tiger_ca_water"
+        "tiger_water_06013",
+        "tiger_water_06001"
     ]
 }

packs/us-ca-sfba.json

@@ -34,6 +34,24 @@
         "nhdplus_h_1804_hu4",
         "nhdplus_h_1805_hu4",
         "nhdplus_h_1806_hu4",
-        "tiger_ca_water"
+        "tiger_water_06085",
+        "tiger_water_06087",
+        "tiger_water_06055",
+        "tiger_water_06081",
+        "tiger_water_06099",
+        "tiger_water_06053",
+        "tiger_water_06097",
+        "tiger_water_06033",
+        "tiger_water_06113",
+        "tiger_water_06013",
+        "tiger_water_06001",
+        "tiger_water_06041",
+        "tiger_water_06075",
+        "tiger_water_06069",
+        "tiger_water_06045",
+        "tiger_water_06047",
+        "tiger_water_06077",
+        "tiger_water_06067",
+        "tiger_water_06095"
     ]
 }

packs/us-ca-tahoe.json

@@ -23,6 +23,13 @@
         "nhdplus_h_1605_hu4",
         "nhdplus_h_1802_hu4",
         "nhdplus_h_1804_hu4",
-        "tiger_ca_water"
+        "tiger_water_32510",
+        "tiger_water_06005",
+        "tiger_water_06057",
+        "tiger_water_06003",
+        "tiger_water_06061",
+        "tiger_water_06017",
+        "tiger_water_32031",
+        "tiger_water_32005"
     ]
 }

packs/us-ca-yosemite.json

@@ -22,6 +22,10 @@
         "nhdplus_h_1605_hu4",
         "nhdplus_h_1804_hu4",
         "nhdplus_h_1809_hu4",
-        "tiger_ca_water"
+        "tiger_water_06043",
+        "tiger_water_06109",
+        "tiger_water_06019",
+        "tiger_water_06051",
+        "tiger_water_06039"
     ]
 }

rocks.py

@@ -309,7 +309,7 @@ def clean_sources(sources):
         shutil.rmtree(work_path)
 
 
-def make_mbtiles(sources, path="./rocks.mbtiles", bbox=None):
+def make_mbtiles(sources, path="./rocks.mbtiles", bbox=None, geojson_path=None):
     """Export rock units into am MBTiles file"""
     mbtiles_cmd = [
         "ogr2ogr",
@@ -323,7 +323,9 @@ def make_mbtiles(sources, path="./rocks.mbtiles", bbox=None):
         "-dsco", "MAXZOOM=14",
         "-dsco", "DESCRIPTION=\"Geological units\""
     ]
-    if bbox:
+    if geojson_path:
+        mbtiles_cmd += ["-clipdst", geojson_path]
+    elif bbox:
         mbtiles_cmd += [
             "-clipdst",
             str(bbox["left"]),
@@ -357,14 +359,15 @@ def make_rocks(
     clean=False,
     path="./rocks.mbtiles",
     procs=NUM_PROCESSES,
-    bbox=None
+    bbox=None,
+    geojson_path=None
 ):
     """Make rocks MBTiles from a collection of sources"""
     make_database()
     if clean:
         clean_sources(sources)
     load_units(sources, clean=clean, procs=procs)
-    mbtiles_path = make_mbtiles(sources, path=path, bbox=bbox)
+    mbtiles_path = make_mbtiles(sources, path=path, bbox=bbox, geojson_path=geojson_path)
     return mbtiles_path
 
 

water.py

@@ -400,7 +400,7 @@ def load_networks(sources, debug=False):
             util.log(f"{source_table_name} doesn't exist, skipping...")
 
 
-def make_mbtiles(sources, path="./water.mbtiles", bbox=None, debug=False):
+def make_mbtiles(sources, path="./water.mbtiles", bbox=None, geojson_path=None, debug=False):
     """Export water into am MBTiles file"""
     if debug:
         util.log(f"water: making mbtiles for sources: {sources}")
@@ -428,14 +428,19 @@ def make_mbtiles(sources, path="./water.mbtiles", bbox=None, debug=False):
             table_name,
             "-a_srs", f"EPSG:{SRID}",
         ]
-        if bbox:
-            cmd += [
-                "-clipdst",
-                str(bbox["left"]),
-                str(bbox["bottom"]),
-                str(bbox["right"]),
-                str(bbox["top"])
-            ]
+        # Don't clip the waterways, useful to see connectivity across the
+        # entire watershed
+        if table_name != WATERWAYS_TABLE_NAME:
+            if geojson_path:
+                cmd += ["-clipdst", geojson_path]
+            elif bbox:
+                cmd += [
+                    "-clipdst",
+                    str(bbox["left"]),
+                    str(bbox["bottom"]),
+                    str(bbox["right"]),
+                    str(bbox["top"])
+                ]
         util.call_cmd(cmd, check=True)
     # 1. Write additional overview layers of perennial ways and large bodies
     waterways_overview_table_name = f"{WATERWAYS_TABLE_NAME}_overview"
@@ -454,7 +459,9 @@ def make_mbtiles(sources, path="./water.mbtiles", bbox=None, debug=False):
         "-nln", waterways_overview_table_name,
         "-a_srs", f"EPSG:{SRID}"
     ]
-    if bbox:
+    if geojson_path:
+        cmd += ["-clipdst", geojson_path]
+    elif bbox:
         cmd += [
             "-clipdst",
             str(bbox["left"]),
@@ -475,7 +482,9 @@ def make_mbtiles(sources, path="./water.mbtiles", bbox=None, debug=False):
         "-nln", waterbodies_overview_table_name,
         "-a_srs", f"EPSG:{SRID}"
     ]
-    if bbox:
+    if geojson_path:
+        cmd += ["-clipdst", geojson_path]
+    elif bbox:
         cmd += [
             "-clipdst",
             str(bbox["left"]),
@@ -557,7 +566,7 @@ def update_imaginary_waterways():
 
 def make_water(
         sources, clean=False, cleandb=False, cleanfiles=False, bbox=None,
-        path="./water.mbtiles", procs=NUM_PROCESSES, debug=False):
+        path="./water.mbtiles", procs=NUM_PROCESSES, debug=False, geojson_path=None):
     """Process and load all water sources and write them to a MBTiles file"""
     if debug:
         util.log("water: making database")
@@ -570,7 +579,7 @@ def make_water(
     update_imaginary_waterways()
     load_watersheds(sources, debug=debug)
     load_networks(sources, debug=debug)
-    return make_mbtiles(sources, path=path, bbox=bbox, debug=debug)
+    return make_mbtiles(sources, path=path, bbox=bbox, geojson_path=geojson_path, debug=debug)
 
 
 if __name__ == "__main__":