geocoding with components and fix tag_line (#160)

* geocoding with components and fix tag_line

* tests for geocoding without country

* fix tests

* use region biasing

* location search with other branch

Author: samuelcolvin

.gitignore

@@ -9,3 +9,4 @@ sandbox/
 nginx/prod/keys/
 activate.prod.*
 activate.beta.*
+.pytest_cache/

tcsocket/app/geo.py

@@ -8,6 +8,7 @@
 ONE_HOUR = 3_600
 NINETY_DAYS = ONE_HOUR * 24 * 90
 IP_HEADER = 'X-Forwarded-For'
+COUNTRY_HEADER = 'CF-IPCountry'
 logger = logging.getLogger('socket.geo')
 
 
@@ -22,7 +23,12 @@ async def geocode(request):
         return
 
     location_str = location_str.strip(' \t\n\r,.')
-    loc_key = 'loc:' + hashlib.md5(location_str.lower().encode()).hexdigest()
+    region = request.headers[COUNTRY_HEADER].lower()
+    if region == 'gb':
+        # https://en.wikipedia.org/wiki/Country_code_top-level_domain#ASCII_ccTLDs_not_in_ISO_3166-1
+        region = 'uk'
+
+    loc_key = 'loc:' + hashlib.md5(f'{location_str.lower()}|{region}'.encode()).hexdigest()
     redis_pool = request.app['redis']
     settings: Settings = request.app['settings']
 
@@ -32,7 +38,7 @@ async def geocode(request):
         loc_data = await redis.get(loc_key)
         if loc_data:
             result = json.loads(loc_data.decode())
-            logger.info('cached geocode result "%s" > "%s"', location_str, result and result['pretty'])
+            logger.info('cached geocode result "%s|%s" > "%s"', location_str, region, result and result['pretty'])
             return result
 
         ip_key = 'geoip:' + ip_address
@@ -48,6 +54,7 @@ async def geocode(request):
             )
         params = {
             'address': location_str,
+            'region': region,
             'key': settings.geocoding_key,
         }
         data = None
@@ -70,6 +77,6 @@ async def geocode(request):
         else:
             result = None
         await redis.setex(loc_key, NINETY_DAYS, json.dumps(result).encode())
-        logger.info('new geocode result "%s" > "%s" (%d from "%s")',
-                    location_str, result and result['pretty'], geo_attempts, ip_address)
+        logger.info('new geocode result "%s|%s" > "%s" (%d from "%s")',
+                    location_str, region, result and result['pretty'], geo_attempts, ip_address)
         return result

tcsocket/app/management.py

@@ -321,3 +321,11 @@ def add_review_fields(conn):
     """
     conn.execute('ALTER TABLE contractors ADD review_rating DOUBLE PRECISION')
     conn.execute('ALTER TABLE contractors ADD review_duration INTEGER NOT NULL DEFAULT 0')
+
+
+@patch
+def resize_tag_line(conn):
+    """
+    resize the tag_line field on contractors to 255 chars
+    """
+    conn.execute('ALTER TABLE contractors ALTER COLUMN tag_line TYPE VARCHAR(255)')

tcsocket/app/models.py

@@ -54,7 +54,7 @@ class Contractor(Base):
     latitude = Column(Float)
     longitude = Column(Float)
 
-    tag_line = Column(String(63))
+    tag_line = Column(String(255))
     primary_description = Column(Text())
 
     extra_attributes = Column(JSONB)

tests/conftest.py

@@ -249,7 +249,8 @@ async def grecaptcha_post_view(request):
 
 async def geocoding_view(request):
     address = request.GET.get('address')
-    request.app['request_log'].append(('geocode', address))
+    region = request.GET.get('region')
+    request.app['request_log'].append(('geocode', f'{address}|{region}'))
     status = 200
     if address == 'SW1W 0EN':
         loc = {

tests/requirements.txt

@@ -3,10 +3,10 @@ flake8==3.5.0
 isort==4.3.3
 pycodestyle==2.3.1
 pyflakes==1.6.0
-pytest==3.3.2
+pytest==3.4.0
 pytest-aiohttp==0.3.0
 pytest-cov==2.5.1
 pytest-isort==0.1.0
 pytest-mock==1.6.3
-pytest-sugar==0.9.0
+pytest-sugar==0.9.1
 pytest-toolbox==0.2

tests/test_public_filtering_views.py

@@ -187,51 +187,72 @@ async def test_distance_filter(cli, db_conn, company, params, con_distances):
     )
 
     url = str(cli.server.app.router['contractor-list'].url_for(company=company.public_key))
-    r = await cli.get(url, params=params, headers={'X-Forwarded-For': '1.1.1.1'})
+    r = await cli.get(url, params=params, headers={'X-Forwarded-For': '1.1.1.1', 'CF-IPCountry': 'GB'})
     assert r.status == 200, await r.text()
     obj = await r.json()
     assert list(map(itemgetter('link', 'distance'), obj['results'])) == con_distances
 
 
 async def test_geocode_cache(cli, other_server, company):
     url = str(cli.server.app.router['contractor-list'].url_for(company=company.public_key))
-    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.1'})
+    country = {'CF-IPCountry': 'GB'}
+    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.1', **country})
     assert r.status == 200, await r.text()
-    assert other_server.app['request_log'] == [('geocode', 'SW1W 0EN')]
+    assert other_server.app['request_log'] == [('geocode', 'SW1W 0EN|uk')]
     obj = await r.json()
     assert {
         'pretty': 'Lower Grosvenor Pl, Westminster, London SW1W 0EN, UK',
         'lat': 51.4980603,
         'lng': -0.14505
     } == obj['location']
 
-    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.2'})
+    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.2', **country})
     assert r.status == 200, await r.text()
-    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.3'})
+    r = await cli.get(url, params={'location': 'SW1W 0EN'}, headers={'X-Forwarded-For': '1.1.1.3', **country})
     assert r.status == 200, await r.text()
-    assert other_server.app['request_log'] == [('geocode', 'SW1W 0EN')]
+    assert other_server.app['request_log'] == [('geocode', 'SW1W 0EN|uk')]
 
 
 async def test_geocode_rate_limit(cli, other_server, company):
     url = str(cli.server.app.router['contractor-list'].url_for(company=company.public_key))
+    country = {'CF-IPCountry': 'GB'}
     for i in range(10):
-        r = await cli.get(url, params={'location': f'SW1W {i}EN'}, headers={'X-Forwarded-For': '1.1.1.1'})
+        r = await cli.get(url, params={'location': f'SW1W {i}EN'}, headers={'X-Forwarded-For': '1.1.1.1', **country})
         assert r.status == 200, await r.text()
-    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.1'})
+    assert len(other_server.app['request_log']) == 10
+    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.1', **country})
     assert r.status == 429, await r.text()
-    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.1'})
+    assert len(other_server.app['request_log']) == 10
+    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.1', **country})
     assert r.status == 429, await r.text()
-    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.2'})
+    assert len(other_server.app['request_log']) == 10
+    r = await cli.get(url, params={'location': 'SW1W 1ENx'}, headers={'X-Forwarded-For': '1.1.1.2', **country})
     assert r.status == 200, await r.text()
     assert len(other_server.app['request_log']) == 11
 
 
 async def test_geocode_error(cli, other_server, company):
     url = str(cli.server.app.router['contractor-list'].url_for(company=company.public_key))
-    r = await cli.get(url, params={'location': '500'}, headers={'X-Forwarded-For': '1.1.1.1'})
+    r = await cli.get(url, params={'location': '500'}, headers={'X-Forwarded-For': '1.1.1.1', 'CF-IPCountry': 'GB'})
     assert r.status == 500, await r.text()
 
 
+async def test_geocode_other_country(cli, other_server, company):
+    r = await cli.get(
+        cli.server.app.router['contractor-list'].url_for(company=company.public_key),
+        params={'location': 'SW1W 0EN'},
+        headers={'X-Forwarded-For': '1.1.1.1', 'CF-IPCountry': 'US'}
+    )
+    assert r.status == 200, await r.text()
+    obj = await r.json()
+    assert {
+        'pretty': 'Lower Grosvenor Pl, Westminster, London SW1W 0EN, UK',
+        'lat': 51.4980603,
+        'lng': -0.14505,
+    } == obj['location']
+    assert other_server.app['request_log'] == [('geocode', 'SW1W 0EN|us')]
+
+
 async def create_labels(db_conn, company):
     await db_conn.execute(
         sa_labels