Merge pull request #37 from GispoCoding/overpass-api

Overpass API import script

.env.example

@@ -1,7 +1,6 @@
 # Api keys for non-open data sources
 FLICKR_API_KEY=insert_api_key_here
 FLICKR_SECRET=insert_secret_here
-OSM_EXTRACTS_API_KEY=insert_api_key_here
 
 # URL of your installation with osmnames
 OSMNAMES_URL=https://geoviz.gispocoding.fi
@@ -13,3 +12,5 @@ SECRET_KEY=please_generate_random_secret_key
 # Hash using https://werkzeug.palletsprojects.com/en/2.0.x/utils/#werkzeug.security.generate_password_hash
 USERNAME=user-name-to-use
 PASSWORD_HASH=pbkdf2:sha256:260000$password-salt$password-hash
+# Plaintext password, only used in development
+PASSWORD=secret-password

.gitignore

@@ -140,9 +140,9 @@ server/swag/
 .jupyter/
 .python_history
 .vscode/
+.idea
 cache/
 data/
 logs/
 maps/
-pgdata/
-osm2pgsql/
+osm2pgsql/

Dockerfile

@@ -11,22 +11,11 @@ RUN apt-get update \
     gdal-bin \
     # For gdal python bindings
     libgdal-dev \
-    # For building osm2pgsql 1.5 (not available in apt)
-    make cmake g++ libboost-dev libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Build osm2pgsql 1.5 (not available in apt)
-WORKDIR /tmp
-RUN git clone https://github.com/openstreetmap/osm2pgsql.git \
-    && mkdir osm2pgsql/build
-WORKDIR /tmp/osm2pgsql/build
-RUN cmake .. \
-    && make \
-    && make install
-
 COPY requirements.txt /tmp/
-RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
 # Add js files required for interactive Kepler
 RUN jupyter nbextension install --py --sys-prefix keplergl \
@@ -66,7 +55,7 @@ ENV NB_USER=analyst \
 FROM common AS geoviz-server
 
 COPY server/requirements-serve.txt /tmp/
-RUN pip install --no-cache-dir --upgrade -r /tmp/requirements-serve.txt
+RUN pip install --no-cache-dir -r /tmp/requirements-serve.txt
 
 # For flask server, looks like pip install uwsgi fails for some reason
 RUN conda install uwsgi

README.md

@@ -56,8 +56,8 @@ If you're *not* running docker, we recommend creating your own conda env, pyenv,
 which contains conda wheels. The last option should make installing all dependencies easier:
 
 ```
-pyenv install miniconda-latest
-pyenv local miniconda-latest
+pyenv install miniconda3-latest
+pyenv local miniconda3-latest
 pip install -r requirements.txt
 ```
 
@@ -87,11 +87,6 @@ If you wish to import data from the Flickr API, fill in your
 [Flickr api key](https://www.flickr.com/services/api/misc.api_keys.html) and secret
 in a `.env` file or the corresponding environment variable.
 
-If you wish to import OSM data faster by using city-specific
-[OSM extracts](https://www.interline.io/osm/extracts/), fill in your
-[OSM extracts api key](https://app.interline.io/products/osm_extracts/orders/new)
-in a `.env` file or the corresponding environment variable.
-
 To get https certificates, you need to add
 your own domain and subdomain in [docker-compose.yml#L17] and your
 AWS access credentials in server/swag/dns-conf/route53.ini , or read [Swag instructions](https://docs.linuxserver.io/general/swag#create-container-via-dns-validation-with-a-wildcard-cert) and change DNSPLUGIN value at [docker-compose.yml#L20] if you are running on a provider

docker-compose.test.yml

@@ -0,0 +1,11 @@
+version: "3"
+
+services:
+  geoviz_test_db:
+    image: postgis/postgis:latest
+    container_name: geoviz_test_db
+    ports:
+      - "5400:5432"
+    environment:
+      - PGDATA=/var/lib/postgresql/data/pgdata
+      - POSTGRES_PASSWORD=postgres

docker-compose.yml

@@ -2,6 +2,7 @@ services:
   osmnames:
     image: klokantech/osmnames-sphinxsearch
     container_name: osmnames
+
   flask:
     image: gispo/geoviz-server
     container_name: flask
@@ -25,6 +26,7 @@ services:
       - SECRET_KEY
       - USERNAME
       - PASSWORD_HASH
+
   dev:
     image: gispo/geoviz-server
     container_name: dev-flask
@@ -34,7 +36,7 @@ services:
       - osmnames
       - postgis
     ports:
-      - "127.0.0.1:5000:5000"
+      - "5000:5000"
     environment:
       - PGHOST=postgis
       - PGPASSWORD=postgres
@@ -47,8 +49,11 @@ services:
       - SECRET_KEY
       - USERNAME
       - PASSWORD_HASH
+      - PASSWORD
+      - DEV_ENV=1
     working_dir: /app/server
     command: /app/server/start-dev.sh
+
   serve:
     image: ghcr.io/linuxserver/swag
     container_name: swag
@@ -66,28 +71,30 @@ services:
       - ./server/swag/:/config
       - /tmp/flask/uwsgi.sock:/tmp/flask/uwsgi.sock
     ports:
-      - 443:443
-      - 80:80
+      - "443:443"
+      - "80:80"
     depends_on:
       - flask
     restart: always
+
   postgis:
     image: postgis/postgis
     container_name: postgis
     volumes:
-      - ./pgdata:/var/lib/postgresql/data/pgdata
+      - geoviz-pgdata:/var/lib/postgresql/data/pgdata
     ports:
-      - "127.0.0.1:5432:5432"
+      - "5432:5432"
     environment:
       - PGDATA=/var/lib/postgresql/data/pgdata
       - POSTGRES_PASSWORD=postgres
+
   notebook:
     image: gispo/geoviz-notebook
     container_name: notebook
     volumes:
       - .:/home/jovyan
     ports:
-      - "127.0.0.1:8888:8888"
+      - "8888:8888"
     depends_on:
       - osmnames
       - postgis
@@ -100,3 +107,6 @@ services:
       - FLICKR_SECRET
       - OSM_EXTRACTS_API_KEY
       - OSMNAMES_URL
+
+volumes:
+  geoviz-pgdata:

export.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
-import logging
 import os
-import sys
 from datasets import DATASETS
 from ipygis import get_connection_url, QueryResult, generate_map
 from slugify import slugify
@@ -13,11 +11,12 @@
 from notebooks.kepler_h3_config import config  # we may use our own custom visualization config
 from osm_tags import tag_filter
 
-IMPORT_LOG_PATH = 'logs'
+from util import create_logger
+
 MAPS_PATH = "server/maps"
 
 parser = argparse.ArgumentParser(description="Create result map for a given city")
-parser.add_argument("city", default="Helsinki", help="City to import")
+parser.add_argument("city_slug", default="helsinki", help="Slug of city to import")
 parser.add_argument("--datasets",
                     default=" ".join([dataset for dataset in DATASETS]),
                     help="Datasets to include in analysis. Default is to use all imported data. E.g. \"osm access kontur\""
@@ -29,31 +28,23 @@
                          " The result map is independent from the analysis database, so you may save a lot of disk space"
                          " by deleting the data if you don't expect to create the map again.")
 args = vars(parser.parse_args())
+
 # slugify city name just in case export was called with non-slug
-city = slugify(args["city"])
+slug = slugify(args["city_slug"])
 datasets_to_export = args["datasets"].split()
 delete = args.get("delete", False)
 
 # log each city separately
-log_file = os.path.join(os.path.dirname(__loader__.path), IMPORT_LOG_PATH, f"{city}.log")
-logging.basicConfig(
-    format='%(asctime)s %(levelname)-8s %(message)s',
-    level=logging.INFO,
-    datefmt='%Y-%m-%d %H:%M:%S')
-logger = logging.getLogger()
-stdout_handler = logging.StreamHandler(sys.stdout)
-file_handler = logging.FileHandler(log_file)
-logger.addFilter(stdout_handler)
-logger.addHandler(file_handler)
+logger = create_logger(slug)
 
 sql_url = get_connection_url(dbname='geoviz')
 engine = create_engine(sql_url)
 schema_engine = engine.execution_options(
-    schema_translate_map={'schema': city}
+    schema_translate_map={'schema': slug}
 )
 session = sessionmaker(bind=schema_engine)()
 
-logger.info(f"Collecting results for {city} with {datasets_to_export}...")
+logger.info(f"Collecting results for {slug} with {datasets_to_export}...")
 
 queries = {
     dataset: session.query(DATASETS[dataset]['model'])
@@ -63,7 +54,7 @@
 if 'osm' in queries:
     queries['osm'] = queries['osm'].filter(tag_filter)
 
-logger.info(f"Running queries for {city} with {datasets_to_export}...")
+logger.info(f"Running queries for {slug} with {datasets_to_export}...")
 results = [
     QueryResult.create(
         query,
@@ -76,7 +67,7 @@
     for dataset, query in queries.items()
 ]
 
-logger.info(f"Creating map for {city} with {datasets_to_export}...")
+logger.info(f"Creating map for {slug} with {datasets_to_export}...")
 weights = [
     DATASETS[dataset]['weight']
     for dataset in datasets_to_export
@@ -90,11 +81,11 @@
 map_path = os.path.join(os.path.dirname(__loader__.path), MAPS_PATH)
 if not os.path.exists(map_path):
     os.mkdir(map_path)
-filename = os.path.join(map_path, f"{city}.html")
+filename = os.path.join(map_path, f"{slug}.html")
 result_map.save_to_html(file_name=filename)
 # delete interim database at the end, we have all the data we need on the map
 if delete:
-    logger.info(f"Deleting analysis database for {city}...")
-    engine.execute(DropSchema(city, cascade=True))
+    logger.info(f"Deleting analysis database for {slug}...")
+    engine.execute(DropSchema(slug, cascade=True))
 
-logger.info(f"--- Datasets {datasets_to_export} for {city} exported to Kepler.gl ---")
+logger.info(f"--- Datasets {datasets_to_export} for {slug} exported to Kepler.gl ---")