Several updates to vagrant setup, added first stab at OSM script.

README.md

@@ -3,7 +3,7 @@ Underfoot is a mobile app for revealing the hydrological and geological world be
 
 Still reading? This repo is mostly for data prep. I used to have some Ionic-based mobile app stuff here but I've given up on Ionic and am currently just tinkering with a native Android app.
 
-## Vagrant setup
+## Vagrant Setup
 You'll need to install [Vagrant](https://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/).
 ```
 git clone https://github.com/kueda/underfoot.git
@@ -38,40 +38,11 @@ python prepare-database.py
 
 # Install node deps for generating MBTiles
 npm install
-npm install -g tl tilelive-postgis tilelive-mbtiles
 
 # Generate MBtiles in the shared Vagrant synced folder
-tl copy -z 7 -Z 14 \
-  'postgis://ubuntu:ubuntu@localhost:5432/underfoot?table=units' \
-  mbtiles:///vagrant/underfoot.mbtiles
-```
-
-## OS X Setup
-
-```
-# Instal GDAL for GIS data processing and PostGIS / PostgreSQL for data storage
-brew install gdal --with-postgres
-brew install postgis nodejs pyenv-virtualenv
-
-# Set up a python virtual environment
-unset PYTHONPATH
-virtualenv venv -p python3 --no-site-packages
-source venv/bin/activate
-
-# Install python deps and some stuff for working with ESRI Arc/Info coverages
-./setup
-
-# Create the database
-python prepare-database.py
-
-# Install node deps for generating MBTiles
-npm install
-npm install -g tl tilelive-postgis tilelive-mbtiles
-
-# Generate MBtiles in the shared Vagrant synced folder
-tl copy -z 7 -Z 14 \
-  'postgis://ubuntu:ubuntu@localhost:5432/underfoot?table=units' \
-  mbtiles:///vagrant/underfoot.mbtiles
+./node_modules/tl/bin/tl.js copy -z 7 -Z 14 \
+  'postgis://vagrant:vagrant@localhost:5432/underfoot?table=units' \
+  mbtiles:///vagrant/underfoot-units.mbtiles
 ```
 
 # Adding Sources

Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-16.04"
+  config.vm.box = "bento/ubuntu-18.04"
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 2048
@@ -14,18 +14,24 @@ Vagrant.configure("2") do |config|
 
   # Modifications to postgres security to allow port forwarding to the host.
   # Obviously this is insecure and only meant to be used in a local setting
-  pg_hba_access_for_host = "host all all 10.0.2.2/32 trust # Support connections from the vagrant host"
   postgresql_conf_access_for_host = "listen_addresses='*'"
+  pgsql_mods = "grep -q -F \"#{postgresql_conf_access_for_host}\" /etc/postgresql/10/main/postgresql.conf  || echo \"#{postgresql_conf_access_for_host}\" >> /etc/postgresql/10/main/postgresql.conf\n"
+  %w(vagrant ubuntu).each do |user|
+    pgsql_mods << "sudo -u postgres psql -c \"DROP ROLE IF EXISTS #{user}; CREATE ROLE #{user} INHERIT LOGIN SUPERUSER PASSWORD '#{user}'\"\n"
+    pg_hba_access_for_local_ipv4 =   "host all #{user} 127.0.0.1/32 trust # Trust all local connections"
+    pg_hba_access_for_host =         "host all #{user} 10.0.2.2/32 trust # Support connections from the vagrant host"
+    [pg_hba_access_for_local_ipv4, pg_hba_access_for_host].each do |pattern|
+      pgsql_mods << "grep -q -F '#{pattern}' /etc/postgresql/10/main/pg_hba.conf || echo '#{pattern}' >> /etc/postgresql/10/main/pg_hba.conf\n"
+    end
+  end
 
   # Provision the vm with the relevant dependencies, modify postgres settings
   # and create an ubuntu superuser
   config.vm.provision "shell", inline: <<-SHELL
     add-apt-repository ppa:ubuntugis/ppa
     apt-get update
-    apt-get install -y git virtualenv nodejs python3 postgis gdal-bin libgdal1-dev unzip
-    sudo -u postgres psql -c "DROP ROLE IF EXISTS ubuntu; CREATE ROLE ubuntu LOGIN SUPERUSER"
-    grep -q -F '#{pg_hba_access_for_host}' /etc/postgresql/9.5/main/pg_hba.conf  || echo '#{pg_hba_access_for_host}' >> /etc/postgresql/9.5/main/pg_hba.conf
-    grep -q -F "#{postgresql_conf_access_for_host}" /etc/postgresql/9.5/main/postgresql.conf  || echo "#{postgresql_conf_access_for_host}" >> /etc/postgresql/9.5/main/postgresql.conf
+    apt-get install -y git virtualenv npm python3 postgis gdal-bin libgdal-dev unzip osmosis
+    #{pgsql_mods}
     service postgresql restart
   SHELL
 end

osm.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+echo ""
+echo "Downloading an export of recent California OSM data (thanks, Geofabrik!)"
+if ! [ -e california-latest.osm.pbf ]
+then
+  curl -o california-latest.osm.pbf http://download.geofabrik.de/north-america/us/california-latest.osm.pbf
+fi
+
+# Make a PostGIS database for the OSM data using the omsosis schema. Note that
+# those osmosis files should  have been installed with osmosis when the VM was
+# provisioned.
+echo ""
+echo "Creating the database and schema"
+dropdb underfoot_ways
+createdb underfoot_ways
+psql -d underfoot_ways -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;'
+psql -d underfoot_ways < /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6.sql
+psql -d underfoot_ways < /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6_bbox.sql
+psql -d underfoot_ways < /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6_linestring.sql
+
+# Load ways from the PBF into the database
+echo ""
+echo "Loading data from the PBF into the database"
+osmosis \
+  --read-pbf california-latest.osm.pbf \
+  --tf accept-ways highway=* \
+  --tf reject-ways service=* \
+  --tf reject-ways footway=sidewalk \
+  --tf reject-ways highway=proposed \
+  --bounding-box left=-123 top=38 right=-122 bottom=37 \
+  --write-pgsql database="underfoot_ways" user="vagrant" password="vagrant"
+
+# Create a table for ways with just names and highway tags
+echo ""
+echo "Creating a table for ways with just names and highway tags"
+psql underfoot_ways -c "CREATE TABLE underfoot_ways AS SELECT id, version, tags -> 'name' AS name, tags -> 'highway' AS highway, linestring FROM ways"
+
+
+# Export ways into the MBTiles using different zoom levels for different types
+echo ""
+echo "Exporting into MBTiles"
+./node_modules/tl/bin/tl.js copy -i underfoot_ways.json -z 7 -Z 14 \
+  "postgis://ubuntu:ubuntu@localhost:5432/underfoot_ways?table=underfoot_ways&query=(SELECT%20*%20from%20underfoot_ways%20WHERE%20highway%20in%20('motorway','motorway_link','primary','trunk'))%20AS%20foo" \
+  mbtiles:///vagrant/underfoot.mbtiles
+./node_modules/tl/bin/tl.js copy -i underfoot_ways.json -z 11 -Z 14 \
+  "postgis://ubuntu:ubuntu@localhost:5432/underfoot_ways?table=underfoot_ways&query=(SELECT%20*%20from%20underfoot_ways%20WHERE%20highway%20in%20('motorway','primary','secondary','motorway_link','trunk'))%20AS%20foo" \
+  mbtiles:///vagrant/underfoot.mbtiles
+./node_modules/tl/bin/tl.js copy -i underfoot_ways.json -z 13 -Z 14 \
+  "postgis://ubuntu:ubuntu@localhost:5432/underfoot_ways?table=underfoot_ways" \
+  mbtiles:///vagrant/underfoot.mbtiles