fix/misc fixes (csirtgadgets#152)

* support `location` field to fix cifv3 geo gatherer
* https://github.com/csirtgadgets/bearded-avenger/blob/master/cif/gatherer/geo.py#L112

* strip tabs and newlines from str fields

* tweak fqdn regex to more align with dns records, a la rfc2181
* ref https://regex101.com/r/FLA9Bv/59

Author: mdavis332

csirtg_indicator/constants.py

@@ -43,7 +43,7 @@
 ]
 
 FIELDS_GEO = [
-    'cc', 'latitude', 'timezone', 'longitude', 'city', 'region'
+    'cc', 'latitude', 'timezone', 'longitude', 'city', 'region', 'location'
 ]
 
 FIELDS_IP = [
@@ -59,12 +59,8 @@
 # http://stackoverflow.com/a/17871737
 RE_IPV6 = re.compile(r'(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))')
 
-# http://goo.gl/Cztyn2 -- probably needs more work
-# http://stackoverflow.com/a/26987741/7205341
-# ^((xn--)?(--)?[a-zA-Z0-9-_@]+(-[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}(--p1ai)?$
-#RE_FQDN = re.compile('^((?!-))(xn--)?[a-z0-9][a-z0-9-_\.]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$')
-# http://stackoverflow.com/questions/14402407/maximum-length-of-a-domain-name-without-the-http-www-com-parts
-RE_FQDN = re.compile(r'^((?!-))(xn--)?[a-z0-9][a-z0-9-_\.]{0,245}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})\.?$')
+# https://regex101.com/r/FLA9Bv/59
+RE_FQDN = re.compile(r'^(?!(?:[\w]+?\.)?\-[\w\.\-]*?)(?![\w\.]+?\-\.(?:[\w\.\-]+?))(?=[\w\.\-]*?[\w\.\-]*?)(?![\w\.\-]{254})(?!(?:\.?[\w\-\.]*?[\w\-]{64,}\.)+?)(?:[\w\-]+?\.)+?(?![^a-zA-Z])[A-Za-z0-9\-]{2,64}(?<!\-)\.?$')
 #RE_URI_SCHEMES = re.compile(r'^(https?|ftp)://')
 RE_URI_SCHEMES = re.compile(r'^(https?|ftp)$')
 RE_EMAIL = re.compile(r"^[-\w+.!#$%&'*\/=?^_`{|}~;]+@[-.0-9a-zA-Z][-.0-9a-zA-Z]*[a-zA-Z]{2,}$")

csirtg_indicator/indicator.py

@@ -15,6 +15,7 @@
 import textwrap
 import json
 import sys
+import re
 if sys.version_info > (3,):
     from urllib.parse import urlparse
     basestring = (str, bytes)
@@ -57,7 +58,7 @@ def __init__(self, indicator=None, version=PROTOCOL_VERSION, **kwargs):
 
             if isinstance(kwargs[k], basestring):
                 # always strip whitespace
-                kwargs[k] = kwargs[k].strip()
+                kwargs[k] = re.sub(r'\r|\t|\n', ' ', kwargs[k]).strip()
 
                 if self._lowercase is True and k != 'reference': # don't lower reference which may be a url
                     kwargs[k] = kwargs[k].lower()

test/test_fqdn.py

@@ -1,5 +1,7 @@
 from csirtg_indicator import Indicator
+from csirtg_indicator.exceptions import InvalidIndicator
 from faker import Faker
+import pytest
 fake = Faker()
 
 
@@ -13,6 +15,37 @@
     'laser-retargeting-server-production.us-east-1-prod-core-edge-public.spongecell.net',
     'example.org.',
     'an0ther.exAmple.orG.',
+    'under_score.com',
+    '_dc-mx.test.someotherdom.com.',
+]
+
+BAD = [
+    'dot.',
+    'this.is-bad-',
+    'this-too.is_bad-.',
+    'this.is.bad-.com',
+    'space com',
+    'a.12E.',
+    '192.168.1.13F',
+    'underscore.c_om',
+    '-dash.com',
+    'dash-.com',
+    'sub.-dash.com',
+    'sub-.dash.com',
+    '-.com',
+    '-com',
+    '.com',
+    'com',
+    'mkyong.t.t.c',
+    'mkyong,com',
+    'mkyong.com/users',
+    'slash.com/',
+    'a.123',
+    'b.123.',
+    'x.XN--VERMGENSBERATUNG-PWBBJALKJSDFWHATLADKDALKJSDFJWHATLEIUDWLAIFU',
+    'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkk.com',
+    'www.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkk.co.uk',
+    'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcde.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.comm',
 ]
 
 
@@ -45,6 +78,11 @@ def test_fqdn_ok():
         d = d.rstrip('.')
         assert e.indicator == d.lower()
 
+def test_fqdn_not_ok():
+    for d in BAD:
+        with pytest.raises(InvalidIndicator):
+            e = Indicator(d)
+
 
 def test_fqdn_subdomain():
     data = [