Mojave pack (#12)

* Added sources for Mojave pack; generalized process_usgs_source function * Bugfix: imposm removal was broken * Fixed bad lithology; added tests to check lithologies in CSV * Added ofr2011_1044 to Mojave pack * Added sim3109 to Mojave pack * Fixed problems with citations data; added test to verify citations data
kueda · Apr 6, 2023 · f8e91c6 · f8e91c6
1 parent 7f4a55b
commit f8e91c6
Show file tree

Hide file tree

Showing 40 changed files with 985 additions and 222 deletions.
diff --git a/packs/us-ca-mojave.json b/packs/us-ca-mojave.json
@@ -10,11 +10,15 @@
     "geojson": {
         "$ref": "file://./us-ca-mojave.geojson"
     },
-    "description": "Mojave National Preserve and surround area",
+    "description": "Mojave National Preserve and surrounding area",
     "id": "us-ca-mojave",
-    "name": "Mojave National Preserve and surround area, CA, USA",
+    "name": "Mojave National Preserve and surrounding area, CA, USA",
     "osm": "http://download.geofabrik.de/north-america/us/california-latest.osm.pbf",
     "rock": [
+        "sim3376",
+        "ofr2011_1044",
+        "sim3206",
+        "sim3109",
         "of2005_1305_ca"
     ],
     "water": [

diff --git a/setup.py b/setup.py
@@ -37,10 +37,11 @@ def setup_imposm():
     work_path = "bin"
     if not os.path.isdir("bin"):
         os.makedirs(work_path)
-    if os.path.isdir(os.path.join(work_path, "imposm")):
-        shutil.rmtree(os.path.join(work_path, "imposm"))
-    else:
-        os.remove(os.path.join(work_path, "imposm"))
+    imposm_path = os.path.join(work_path, "imposm")
+    if os.path.isdir(imposm_path):
+        shutil.rmtree(imposm_path)
+    elif os.path.isfile(imposm_path):
+        os.remove(imposm_path)
     os.chdir(work_path)
     basename = "imposm-0.11.1-linux-x86-64"
     run([

diff --git a/sources/i1874/__init__.py b/sources/i1874/__init__.py
@@ -13,7 +13,7 @@ def run():
         base_path=os.path.realpath(__file__),
         url="https://pubs.usgs.gov/imap/i1874/yosenp.e00.zip",
         use_unzip=True,
-        e00_path="yosenp.e00",
+        extracted_file_path="yosenp.e00",
         skip_polygonize_arcs=True,
         srs=NAD27_UTM11_PROJ4,
         metadata_csv_path=os.path.join(

diff --git a/sources/mf2337c/__init__.py b/sources/mf2337c/__init__.py
@@ -12,7 +12,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/mf/2000/2337/mf2337c.tgz",
-        e00_path="mageo/ma-geol.e00",
+        extracted_file_path="mageo/ma-geol.e00",
         polygon_pattern="GEOL5-I",
         srs="+proj=lcc +lat_1=37.06666666666 +lat_2=38.43333333333 "
             "+lat_0=36.5 +lon_0=-120.5 +x_0=90 +y_0=10 +ellps=clrk66 +units=m "

diff --git a/sources/mf2342c/__init__.py b/sources/mf2342c/__init__.py
@@ -13,7 +13,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/mf/2000/2342/mf2342c.tgz",
-        e00_path="oakdb/*-geol.e00",
+        extracted_file_path="oakdb/*-geol.e00",
         polygon_pattern="PY#",
         srs=STATE_PLANE_CA_ZONE_3,
         metadata_csv_path=os.path.join(

diff --git a/sources/mf2402/__init__.py b/sources/mf2402/__init__.py
@@ -13,7 +13,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/mf/2002/2402/mf2402c.tgz",
-        e00_path="wsogeo/wso-geol.e00",
+        extracted_file_path="wsogeo/wso-geol.e00",
         srs=NAD27_UTM10_PROJ4,
         metadata_csv_path=os.path.join(
             os.path.dirname(os.path.realpath(__file__)),

diff --git a/sources/mf2403c/__init__.py b/sources/mf2403c/__init__.py
@@ -12,7 +12,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/mf/2002/2403/mf2403c.tgz",
-        e00_path="nesfgeo/*-geol.e00",
+        extracted_file_path="nesfgeo/*-geol.e00",
         srs=STATE_PLANE_CA_ZONE_3,
         polygon_pattern="GEOL",
         metadata_csv_path=os.path.join(

diff --git a/sources/of2005_1019/__init__.py b/sources/of2005_1019/__init__.py
@@ -11,7 +11,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="https://pubs.usgs.gov/of/2005/1019/los_angeles.tar.gz",
-        e00_path="los_angeles/la1_geo.e00",
+        extracted_file_path="los_angeles/la1_geo.e00",
         polygons_join_col="LABL",
         skip_polygonize_arcs=True,
         srs=NAD27_UTM11_PROJ4,

diff --git a/sources/of94_622/__init__.py b/sources/of94_622/__init__.py
@@ -12,7 +12,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1994/of94-622/cc_g1.tar.Z",
-        e00_path="ccgeo/cc_utm",
+        extracted_file_path="ccgeo/cc_utm",
         polygon_pattern="CC_UTM#",
         srs=NAD27_UTM10_PROJ4,
         metadata_csv_path=os.path.join(

diff --git a/sources/of95_597/__init__.py b/sources/of95_597/__init__.py
@@ -13,7 +13,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1995/of95-597/hf_g1.tar.Z",
-        e00_path="hfg/hf*_um",
+        extracted_file_path="hfg/hf*_um",
         srs=NAD27_UTM10_PROJ4,
         skip_polygonize_arcs=True
     )
diff --git a/sources/of96_252/__init__.py b/sources/of96_252/__init__.py
@@ -13,7 +13,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1996/of96-252/al_g1.tar.Z",
-        e00_path="algeo/al_um-py.e00",
+        extracted_file_path="algeo/al_um-py.e00",
         srs=NAD27_UTM10_PROJ4,
         polygon_pattern="AL_UM-PY#",
         metadata_csv_path=os.path.join(

diff --git a/sources/of97_456/__init__.py b/sources/of97_456/__init__.py
@@ -8,7 +8,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1997/of97-456/pr-data.tar.gz",
-        e00_path="pr-data/pr-geol.e00",
+        extracted_file_path="pr-data/pr-geol.e00",
         srs=util.NAD27_UTM10_PROJ4,
         metadata_csv_path=os.path.join(
             os.path.dirname(os.path.realpath(__file__)),

diff --git a/sources/of97_489/__init__.py b/sources/of97_489/__init__.py
@@ -8,7 +8,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1997/of97-489/sc-geol.e00.gz",
-        e00_path="sc-geol.e00",
+        extracted_file_path="sc-geol.e00",
         srs=util.NAD27_UTM10_PROJ4,
         metadata_csv_path=os.path.join(
             os.path.dirname(os.path.realpath(__file__)),

diff --git a/sources/of98_137/__init__.py b/sources/of98_137/__init__.py
@@ -8,7 +8,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1998/of98-137/sm_g1.tar.gz",
-        e00_path="smgeo/sm_um-py.e00",
+        extracted_file_path="smgeo/sm_um-py.e00",
         polygon_pattern="SM3_UM-PY",
         srs=util.NAD27_UTM10_PROJ4,
         metadata_csv_path=os.path.join(

diff --git a/sources/of98_354/__init__.py b/sources/of98_354/__init__.py
@@ -9,7 +9,7 @@ def run():
         base_path=os.path.realpath(__file__),
         url="http://pubs.usgs.gov/of/1998/of98-354/sfs_data.tar.gz",
         uncompress_e00=True,
-        e00_path="sfs-geol.e00",
+        extracted_file_path="sfs-geol.e00",
         srs="+proj=lcc +lat_1=37.066667 +lat_2=38.433333 +lat_0=36.5 "
             "+lon_0=-120.5 +x_0=609601.21920 +y_0=-6 +datum=NAD27 +units=m "
             "+no_defs",

diff --git a/sources/of99_014/__init__.py b/sources/of99_014/__init__.py
@@ -13,7 +13,7 @@ def run():
     process_usgs_source(
         base_path=os.path.realpath(__file__),
         url="https://pubs.usgs.gov/of/1999/of99-014/of99-14_3a.e00.gz",
-        e00_path="of99-14_3a.e00",
+        extracted_file_path="of99-14_3a.e00",
         polygon_pattern="CARR-GEOL#",
         # Supposed to be California State Plane Zone 5
         # (https://spatialreference.org/ref/epsg/2229/), but this works

diff --git a/sources/ofr2011_1044.py b/sources/ofr2011_1044.py
@@ -0,0 +1,9 @@
+"""
+Preliminary Surficial Geologic Map of the Newberry Springs 30’ x 60’
+Quadrangle, California
+"""
+
+from ofr2011_1044 import run
+
+if __name__ == "__main__":
+    run()
diff --git a/sources/ofr2011_1044/__init__.py b/sources/ofr2011_1044/__init__.py
@@ -0,0 +1,28 @@
+"""
+Preliminary Surficial Geologic Map of the Newberry Springs 30’ x 60’
+Quadrangle, California
+"""
+
+import os
+import re
+
+from util.rocks import process_usgs_source
+from util.proj import NAD27_UTM11_PROJ4
+
+
+def run():
+    process_usgs_source(
+        base_path=os.path.realpath(__file__),
+        url="https://pubs.usgs.gov/of/2011/1044/OFR2011-1044_database.zip",
+        extracted_file_path="OFR2011-1044_database/OFR2011-1044.e00",
+        skip_polygonize_arcs=True,
+        srs=NAD27_UTM11_PROJ4,
+        use_unzip=True,
+        # This dataset specifies units using codes for surficial rocks AND
+        # basement rocks. Here we're just using the stuff at the surface.
+        join_col_modifier=lambda ptype: re.split(r'[/+-]', ptype)[0],
+        metadata_csv_path=os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            "units.csv"
+        )
+    )
diff --git a/sources/ofr2011_1044/citation.json b/sources/ofr2011_1044/citation.json
@@ -0,0 +1,48 @@
+[
+	{
+		"id": "http://zotero.org/users/local/b90sdi4a/items/UU38RG2Y",
+		"type": "map",
+		"publisher": "U.S. Geological Survey",
+		"scale": "1:100,000",
+		"title": "Preliminary Surficial Geologic Map of the Newberry Springs 30' x 60' Quadrangle, California",
+		"URL": "https://pubs.usgs.gov/of/2011/1044/",
+		"contributor": [
+			{
+				"family": "Phelps",
+				"given": "G.A."
+			},
+			{
+				"family": "Bedford",
+				"given": "D.R."
+			},
+			{
+				"family": "Lidke",
+				"given": "D.J."
+			},
+			{
+				"family": "Miller",
+				"given": "D.M."
+			},
+			{
+				"family": "Schmidt",
+				"given": "K.M."
+			}
+		],
+		"accessed": {
+			"date-parts": [
+				[
+					"2023",
+					3,
+					18
+				]
+			]
+		},
+		"issued": {
+			"date-parts": [
+				[
+					"2012"
+				]
+			]
+		}
+	}
+]