Replace streetviewfetch folder with new streetview_retrieval

Author: pykenny

streetview_retrieval/dataset_convert.py

@@ -0,0 +1,38 @@
+""" dataset_convert.py
+    Author: Po-Yu Hsieh ([email protected])
+    Last update: 2019/05/02
+
+"""
+import sys
+import tools.coordinate_conversion as corcv
+
+def run():
+    if len(sys.argv) != 5:
+        print("Usage: dataset_convert.py [STREETDB_IN] [SIDEWALKDB_IN] [STREETDB_OUT] [SIDEWALKDB_OUT]")
+        return
+
+    _STREET_DB_PATH, _SIDEWALK_DB_PATH, _STREET_CONV_OUT_PATH, _SIDEWALK_CONV_OUT_PATH = sys.argv[1:]
+    _SRC_CODE, _TGT_CODE = 6492, 4326  # EPSG codes
+
+    _PARAMS_ST = {
+        "input_file": _STREET_DB_PATH, "output_file": _STREET_CONV_OUT_PATH,
+        "source_code": _SRC_CODE, "target_code": _TGT_CODE, "xy_coordinate": True,
+        "record_map": lambda rec: {"name": rec["GREENBOOK"]}
+    }
+
+    _PARAMS_SW = {
+        "input_file": _SIDEWALK_DB_PATH, "output_file": _SIDEWALK_CONV_OUT_PATH,
+        "source_code": _SRC_CODE, "target_code": _TGT_CODE, "xy_coordinate": True
+    }
+
+    print("Start converting street dataset coordinates...")
+    toolkit = corcv.CoordinateConversionToolset(**_PARAMS_ST)
+    toolkit.convert_shape_coordinates()
+    print("Start converting sidewalk dataset coordinates...")
+    toolkit = corcv.CoordinateConversionToolset(**_PARAMS_SW)
+    toolkit.convert_shape_coordinates()
+    print("Done!")
+
+
+if __name__ == "__main__":
+    run()

streetview_retrieval/dependencies.txt

@@ -0,0 +1,6 @@
+requests
+pyshp
+rtree
+nvector
+geopy
+PyGeodesy

streetview_retrieval/query_generation.py

@@ -0,0 +1,43 @@
+import argparse
+import tools.sidewalk_partition as sp
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Sidewalk partition and query parameter \
+                                     generation routine for Boston StreetCaster project.")
+    """
+        Required Arguments
+    """
+    parser.add_argument("--sidewalkfile", dest="sidewalk_file", type=str, required=True,
+                        help="File path to sidewalk JSON dataset.")
+    parser.add_argument("--index_path", dest="index_path", type=str, required=True,
+                        help="File path to store R-Tree index of street segments.")
+    parser.add_argument("--out_path", dest="out_path", type=str, required=True,
+                        help="Output folder path to store query parameters and metadata.")
+
+    """
+        Optional Arguments
+    """
+    parser.add_argument("--street_file", dest="street_file", type=str, required=False,
+                        help="If R-tree index is not built, then this is the path for converted \
+                        street coordinate file used to construct spatial index.")
+    parser.add_argument("--part_len", dest="part_len", type=float, required=False,
+                        help="Partition lengh for sidewalk blocks (in meters).")
+    parser.add_argument("--threshold", dest="threshold", type=float, required=False,
+                        help="Threshod for identifying matching street segment (in degrees).")
+    parser.add_argument("--shot_angle", dest="shot_angle", type=float, required=False,
+                        help="Camera angle difference to sidwalk's direction (in degrees).")
+    parser.add_argument("--shot_dist", dest="shot_dist", type=float, required=False,
+                        help="Camera distance to sidewalk partition's center (in meters).")
+    parser.add_argument("--verbose", dest="verbose", required=False, action="store_true",
+                        help="Print out query parameter settings for each partition.")
+
+    return {key: val for key, val in vars(parser.parse_args()).items() if val}
+
+def run():
+    """ Main routine.
+    """
+    arguments = parse_args()
+    sp.QueryGenerationRunner(**arguments).run()
+
+if __name__ == "__main__":
+    run()

streetview_retrieval/query_sampling.py

@@ -0,0 +1,104 @@
+""" query_sampling.py
+
+    Author: Po-Yu Hsieh ([email protected])
+    Last Update: 2019/05/02
+"""
+
+import os
+import sys
+import json
+import csv
+from random import randint, sample
+from copy import deepcopy
+import tools.streetview_retrieval as sr
+
+
+def query_count(filepath):
+    with open(filepath, "r") as fd_r:
+        result = len(fd_r.readlines())
+    return result
+
+def sample_runner(file_queries, file_meta, sample_size, credential_path, output_info=None, subsample=None):
+    print("Initialize search tool...")
+    qrtool = sr.StreetviewQueryToolset(credential_path=credential_path, verbose=True)
+    query_data_m = {
+        "location": "",
+        "size": "640x420",
+        "heading": 0,
+        "fov": 90,
+        "pitch": 0,
+        "radius": 20,
+        "source": "outdoor"
+    }
+    query_data_s = {
+        "pano": "",
+        "size": "640x420",
+        "heading": 0,
+        "fov": 90,
+        "pitch": 0,
+        "radius": 20,
+        "source": "outdoor"
+    }
+
+    k = sample_size
+    if subsample and subsample > k:
+        sample_list = sample(range(query_count(file_queries)), subsample)
+        subsample = True
+    else:
+        subsample = False
+
+    buckets = [None for i in range(k)]
+    stream_ct = 0
+
+    print("Sampling...")
+    with open(file_meta, newline='') as meta_csv, open(file_queries) as query_file:
+        m_reader = csv.DictReader(meta_csv)
+        for row_id, (m_dict, q_line) in enumerate(zip(m_reader, query_file.readlines())):
+            # Step 0: Work under subsampling
+            if subsample and (row_id not in sample_list):
+                continue
+            # Step 1: Form parameters and check data availability
+            print("Handling data id: {}".format(row_id))
+            query_json = json.loads(q_line)
+            query_data_m["location"] = "{0:f},{1:f}".format(*(query_json["location"][::-1]))
+            query_data_m["heading"] = query_json["heading"]
+            pano_id = qrtool.get_meta(query_data_m)
+
+            if pano_id:
+                query_data_s["pano"] = pano_id
+                query_data_s["heading"] = query_json["heading"]
+                # Step 2-1: For the first k ones, insert them anyway
+                if stream_ct < k:
+                    buckets[stream_ct] = {"row_id": row_id, "meta": m_dict, "query": deepcopy(query_data_s)}
+                # Step 2-2: For items afterward, do resevoir sampling
+                else:
+                    rv_decision = randint(0, stream_ct)
+                    if rv_decision < k:
+                        buckets[rv_decision] = {"row_id": row_id, "meta": m_dict, "query": deepcopy(query_data_s)}
+                # Increment counter
+                stream_ct += 1
+
+        # Step 3: Make queries based on sampled result
+        print("Query based on sample result...")
+        # For safety.
+        if output_info:
+            fd_w = open(output_info, "w")
+            for q_item in buckets:
+                fd_w.write("{}\n".format(json.dumps(q_item)))
+            fd_w.close()
+        print("Done!")
+
+def run():
+    if not 5 <= len(sys.argv) <= 6:
+        print("Usage: python3.7 query_sampling.py QUERY_FILE META_FILE SAMPLE_SIZE AUTH_PATH OUTPUT_INFO_PATH ([SUBSAMPLE_SIZE])")
+        return
+    file_queries, file_meta, sample_size, credential_path, output_info = sys.argv[1:6]
+    sample_size = int(sample_size)
+    if len(sys.argv) == 6:
+        subsample = int(sys.argv[6])
+    else:
+        subsample = None
+    sample_runner(file_queries, file_meta, sample_size, credential_path, output_info, subsample)
+
+if __name__ == "__main__":
+    run()

streetview_retrieval/readme.md

@@ -0,0 +1,104 @@
+# Sidewalk Dataset Preprocessing
+
+In this document we'll describe how we preprocess dataset we're given, and derive what we needed for querying Google's [street view API](https://developers.google.com/maps/documentation/streetview/intro) for sidewalk images.
+
+## About Dataset and Resource
+
+We have our sidewalk and street data from Boston Hack 2018's dataset. The whole dataset can be obtained from project description [here](https://docs.google.com/document/d/1jS3QsgjQLZyYoZzs0WbrA_SrOWAhEUv6Cc_a8X0oHJA/).
+
+The street dataset consists of line segments of streets in Boston; and sidewalk contains shape polygons of planned sidewalk blocks. Both datasets use coordinates under *EPSG:6492*'s ([description](https://epsg.io/6492)) projection settings. 
+
+### Coordinate Conversion
+
+Because Google's street view API only accept *WGS84* (worldwide coordinate system, [ref](https://en.wikipedia.org/wiki/World_Geodetic_System)), while the dataset uses another coordinate system, we need to transform location coordinates beforehand. There are several Python libraries supporting coordinate conversion, such as `pyproj`. However, we found that transfomation result from local machine suffers from significant numeral error, which can't be directly applied in any further task.
+
+To address this issue, we use [epsg.io](http://epsg.io/)'s free API to obtain coordinates with higher precision ([API description](https://github.com/klokantech/epsg.io)). We wrote a function that converts a series of locations from one projection system to the other for this task.
+
+### Sidewalk Block Partition
+
+Most of the sidewalk blocks are too long to be included in single image, hence we may want to separate single block into multiple partitions of similar length, then retrieve images separately. So the next thing to do is to know about how many photos to be taken along the sidewalk, along with location of each partition block. We desinged an algorithm to approximate a simpler shape of the sidewalk, then do partition on the simplified shape to obtain requried partitions.
+
+The approximation algorithm is based on the assumpiton that most of the blocks are neary rectangular-shaped. Firstable, we approxmiate the block as a quadrilateral defined by points which obtain extreme value (maximum and minimum) on both coordinate directions. Then, take the longest side of the quadrilateral to decide number of partitions and main heading direction to the block. Last we partitioned the longest edge along with the edge across, and take the midpoint of evenly segmented points on both sides as partition's center.
+
+### Street Matching and Camera Settings
+
+The main problem of the dataset is that the sidewalk dataset doesn't provide any information about which road it belongs to. Also, we need to know relative position between street and sidewalk to make sure the camera is set up on the correct side of target sidewalk in order to obtain valid result. In spatial database applications, the most basic way for various kinds of searching operations (such as intersection and nearest-neighbor search) is to build up index for regtangualr bounding boxes of all itmes, then work on relationship between bounding boxes before any analysis on detailed shapes inside. Another useful clue that can be used to match street segment and sidewalk is using angle difference between their heading directions, since idealy they're nearly parallel to each other.
+
+So this is how we deal with this case. First, we build up an R-tree index for all line segments in the street dataset, then use the center of target sidewalk partition to do nearest-neighbor query (on bounding boxes) for possible candidates, then use angle differenceto find out segment that best fits.
+
+### Obtaining Sidewalk Images from Google's Street View API
+
+Other than solving camera heading, we need to decide other factors such as camera position, pitch rotation, and zooming for querying street view images, while guarantee the area we want appears is large enough on result image. However, obtaining the right camera settings is another researchable problem far beyond our main goal, so we just set all camera positions to be some point away from partition centner at a fixed distance, and use default value in Google's API for other settings.
+
+### Future Work
+
+We're told that the original datset for sidewalks [Link](https://data.boston.gov/dataset/sidewalk-inventory) has more attributes that are crucial to refine all above tasks, such as street it belongs to and sidewalk width. Future updates will combine this dataset so that we can obtain more accurate camera position and settings for the target.
+
+Also, the assumptions we used for generating camera settings can fail in some cases, and there are still lots of possible improvement can be done in the future, such as approximating parameters other than position and heading for the query. 
+
+### Dependencies
+
+All the programs are developed under Python3.7, and requre these external libraries other than built-ins:
+
+* `requests`, library for HTTP requests
+* `pyshp`, library for reading Shapefile
+* `rtree`, wrapper library for [libspatialindex](https://libspatialindex.org/) to build up R-tree index.
+* `nvector`, library that provides tools for solving common geographical questions
+* `geopy`, library for geocoding and distance computation
+* `PyGeodesy`, library for geodesy operations
+
+First, run this command to install all dependencies:
+
+```
+python3.7 -m pip install -r dependencies.txt
+```
+
+Then follow [this](http://toblerity.org/rtree/install.html) instruction to have `libspatialindex` installed. For macOS users (which is not covered from official website), run `brew install spatialindex` through `homebrew` to install this package.
+
+### How to run
+#### Coordination Conversion
+
+Say you have your Shapefiles for street and sidewalk in directories `ST_DB` and `SW_DB`, and want to output the converted result to `ST_OUT_PATH`, `SW_OUT_PATH`, respectively. Then you can run this command to convert both datasets:
+```
+python3 dataset_convert.py "ST_DB" "SW_DB" "ST_OUT_PATH" "SW_OUT_PATH"
+```
+
+#### Partitioning, Query Generation and Obtain Images
+
+To partition converted sidewalk datset and generate query parameters, use `query_generation.py` and run:
+
+```
+python3.7 query_generation.py [YOUR_PARAMETERS...]
+```
+
+For details about parameters, check them out by running:
+
+```
+python3.7 query_generation.py -h
+```
+
+Running `query_generation` will output three text files:
+- `sidewalk_info.txt`: Information of sidewalk blocks, mainly about how many partitions are made for this block for querying.
+- `metadata.txt`: Metadata for partition queries, includeing their center position, heading direction and the matched street segment.
+- `queries.txt`: Query parameters for Google street view API (in JSON string format)
+
+As for obtaining sidewalk from queries generated, right now we haven't solution to obtain. Since Google's street view API is chraged service, user may consider not requesting all images at once.
+
+However, if you'd like to sample from the set of all queries, then you can try `sample_retrive.py` and `query_sampling.py`, which samples from output result above, filter out ones with valid result from Google's API, and query street view images from the samples.
+
+Google's API require product key and signature secret for forming queries, so you need to store your authetication credentials in a JSON file with this format:
+
+```json
+{
+  "key": "[Your Google API key here]",
+  "secret": "[And your API signature secret here]"
+}
+```
+
+Suppose that you store all outputs from `query_generation` under `partitions` directory, and store the keys in `credentials.json`. Then running the following commands will randomly select 100 valid queries (out of 1000 samples) and store images obtained to `sample_output` directory (with *test_sample_* as filename prefix): 
+
+```
+python3.7 query_sampling.py partitions/queries.txt partitions/metadata.txt 100 credentials.json partitions/samples.txt 1000
+python3.7 sample_retrive.py partitions/samples.txt credentials.json sample_output test_sample_ [-v]
+```
+

streetview_retrieval/sample_retrive.py

@@ -0,0 +1,49 @@
+""" sample_retrive.py
+
+    Author: Po-Yu Hsieh ([email protected])
+    Last Update: 2019/05/02
+"""
+
+import os, sys
+import json
+import tools.streetview_retrieval as sr
+
+def fetch_sidewalk_images(query_file, auth_path, output_path, tag, verbosity=False):
+    """ Obtain images from sample information provided in run.py.
+
+        Args:
+            query_file - (str) File path for query information.
+            auth_path - (str) File path for Google API's auth keys.
+            out_path - (str) Output path for returned images.
+            tag - (str) Prefix for output images
+            verbosity - (bool) Verbosity settings for StreetviewQueryToolset
+    """
+    qrtool = sr.StreetviewQueryToolset(credential_path=auth_path, verbosity=verbosity)
+    with open(query_file, "r") as fd_r:
+        for line in fd_r.readlines():
+            query_params = json.loads(line)["query"]
+            # Even if it's Google we can still get errors, in this case we just retry it.
+            while True:
+                try:
+                    qrtool.get_streetview(output_path, json.loads(line)["query"], tag, False)
+                except:
+                    continue
+                else:
+                    break
+
+def run():
+    if len(sys.argv) < 5 or (len(sys.argv) == 6 and "-v" not in sys.argv):
+        print("Usage:\n  python3.7 sample_retrive.py QUERY_PATH AUTH_PATH OUT_PATH FILE_PREFIX [-v]")
+        return
+    arguments = [i for i in sys.argv[1:]]
+    if len(argument) == 5:
+        _ = argument.pop(argument.index("-v"))
+        verbosity = True
+    else:
+        verbosity = False
+    query_file, auth_path, out_path, tag = arguments
+    fetch_sidewalk_images(query_file, auth_path, out_path, tag, verbosity)
+
+
+if __name__ == "__main__":
+    run()