Merge pull request #495 from alephdata/rinat/compare-entities

Entity comparison function

Author: pudo

aleph/index/xref.py

@@ -64,7 +64,7 @@ def entity_query(sample, collection_id=None, query=None, broad=False):
                 'names.text': {
                     'query': name,
                     'operator': 'and',
-                    'minimum_should_match': '67%',
+                    'minimum_should_match': '5%',
                     # 'cutoff_frequency': 0.0001,
                     # 'boost': 0.5
                 }

aleph/logic/compare.py

@@ -0,0 +1,60 @@
+import itertools
+from Levenshtein import jaro
+from banal import ensure_list
+from followthemoney import model
+from followthemoney.types import registry
+from followthemoney.util import dampen
+
+# OK, Here's the plan: we have to find a way to get user judgements
+# on as many of these matches as we can, then build a regression
+# model which properly weights the value of a matching property
+# based upon it's type.
+FP_WEIGHT = 0.6
+MATCH_WEIGHTS = {
+    registry.text: 0,
+    registry.name: 0,  # because we already compare fingerprints
+    registry.identifier: 0.4,
+    registry.url: 0.1,
+    registry.email: 0.3,
+    registry.ip: 0.1,
+    registry.iban: 0.3,
+    registry.address: 0.2,
+    registry.date: 0.3,
+    registry.phone: 0.1,
+    registry.country: 0.1,
+    registry.language: 0.1,
+}
+
+
+def compare(left, right):
+    """Compare two entities and return number between 0 and 1.
+    Returned number indicates probability that two entities are the same.
+    """
+    left_schema = model.get(left.get('schema'))
+    right_schema = model.get(right.get('schema'))
+    if right_schema not in list(left_schema.matchable_schemata):
+        return 0
+    schema = model.precise_schema(left_schema, right_schema)
+    score = compare_fingerprints(left, right)
+    left_properties = left.get('properties', {})
+    right_properties = right.get('properties', {})
+    for name, prop in schema.properties.items():
+        weight = MATCH_WEIGHTS.get(prop.type, 0)
+        if weight == 0:
+            continue
+        left_values = left_properties.get(name)
+        right_values = right_properties.get(name)
+        prop_score = prop.type.compare_sets(left_values, right_values)
+        score = score + prop_score * weight
+    return max(0.0, min(1.0, score)) * 0.9
+
+
+def compare_fingerprints(left, right):
+    result = 0
+    left_list = ensure_list(left.get('fingerprints'))
+    right_list = ensure_list(right.get('fingerprints'))
+    for (left, right) in itertools.product(left_list, right_list):
+        similarity = jaro(left, right)
+        score = similarity * dampen(3, 20, min(left, right, key=len))
+        result = max(result, score)
+    return result * FP_WEIGHT

aleph/logic/xref.py

@@ -7,12 +7,14 @@
 from aleph.index.core import entities_index
 from aleph.index.xref import entity_query
 from aleph.index.entities import iter_entities
-from aleph.index.util import search_safe
+from aleph.index.util import search_safe, unpack_result
+from aleph.logic.compare import compare
 
 log = logging.getLogger(__name__)
+EXCLUDES = ['text', 'roles']
 
 
-def _xref_item(item, collection_id=None):
+def xref_item(item, collection_id=None):
     """Cross-reference an entity or document, given as an indexed document."""
     name = item.get('name') or item.get('title')
     query = entity_query(item, collection_id=collection_id)
@@ -21,8 +23,8 @@ def _xref_item(item, collection_id=None):
 
     query = {
         'query': query,
-        'size': 10,
-        '_source': ['collection_id', 'name'],
+        'size': 15,
+        '_source': {'excludes': EXCLUDES}
     }
     result = search_safe(index=entities_index(), body=query)
     results = result.get('hits').get('hits')
@@ -40,16 +42,16 @@ def _xref_item(item, collection_id=None):
     dq.delete()
 
     for result in results:
-        source = result.get('_source', {})
-        log.info("Xref [%.1f]: %s <=> %s", result.get('_score'),
-                 name, source.get('name'))
+        result = unpack_result(result)
+        score = compare(item, result)
+        log.info("Xref [%.1f]: %s <=> %s", score, name, result.get('name'))
         obj = Match()
         obj.entity_id = entity_id
         obj.document_id = document_id
         obj.collection_id = item.get('collection_id')
-        obj.match_id = result.get('_id')
-        obj.match_collection_id = source.get('collection_id')
-        obj.score = result.get('_score')
+        obj.match_id = result.get('id')
+        obj.match_collection_id = result.get('collection_id')
+        obj.score = score
         db.session.add(obj)
     db.session.commit()
 
@@ -60,6 +62,6 @@ def xref_collection(collection_id, other_id=None):
     matchable = [s.name for s in model if s.matchable]
     entities = iter_entities(collection_id=collection_id,
                              schemata=matchable,
-                             excludes=['text', 'roles', 'properties.*'])
+                             excludes=EXCLUDES)
     for entity in entities:
-        _xref_item(entity, collection_id=other_id)
+        xref_item(entity, collection_id=other_id)

platform/base/Dockerfile

@@ -100,4 +100,3 @@ RUN python3 -m spacy download xx
 # RUN python3 -m spacy download es && python3 -m spacy download pt
 # RUN python3 -m spacy download fr && python3 -m spacy download it
 # RUN python3 -m spacy download nl
-

requirements-generic.txt

@@ -65,4 +65,4 @@ python-magic==0.4.15
 odfpy==1.3.6
 olefile==0.45.1
 -e git://github.com/alephdata/flanker.git@65eb960f1eaa1fe2925b03f3740e2eff3c21dafc#egg=flanker
-ply==3.10
+ply==3.10

ui/src/screens/CollectionXrefMatchesScreen/CollectionXrefMatchesScreen.jsx

@@ -146,7 +146,7 @@ class CollectionXrefMatchesScreen extends Component {
             { matches.total !== undefined && matches.results.map((match) => (
               <tr key={match.id}>
                 <td className="numeric narrow">
-                  <FormattedNumber value={parseInt(match.score, 10)} />
+                  <FormattedNumber value={parseInt(parseFloat(match.score) * 100, 10)} />
                 </td>
                 {match.entity && (
                   <React.Fragment>