Catalogue concept ES modifications

Author: StepanBrychta

catalogue_graph/src/ingestor_loader.py

@@ -36,10 +36,13 @@ def extract_data(start_offset: int, end_index: int, is_local: bool) -> list[dict
 
     open_cypher_match_query = f"""
     MATCH (concept:Concept)
-    OPTIONAL MATCH (concept)-[:HAS_SOURCE_CONCEPT]->(linked_source_concept)-[:SAME_AS*0..]->(source_concept)
-    RETURN concept, collect(DISTINCT source_concept) AS source_concepts
-    ORDER BY concept.id
+    WITH concept ORDER BY concept.id
     SKIP {start_offset} LIMIT {limit}
+    OPTIONAL MATCH (concept)-[:HAS_SOURCE_CONCEPT]->(linked_source_concept)-[:SAME_AS*0..]->(source_concept)
+    RETURN 
+        concept,
+        collect(DISTINCT linked_source_concept) AS linked_source_concepts,
+        collect(DISTINCT source_concept) AS source_concepts
     """
 
     print("Running query:")

catalogue_graph/src/models/catalogue_concept.py

@@ -6,7 +6,7 @@
 
 def get_priority_source_concept_value(values: dict) -> Any:
     # Sources sorted by priority
-    for source in ["nlm-mesh", "lc-names", "lc-subjects", "wikidata"]:
+    for source in ["nlm-mesh", "lc-names", "lc-subjects", "wikidata", "label-derived"]:
         if (value := values.get(source)) is not None:
             return value
 
@@ -26,15 +26,20 @@ class CatalogueConcept(BaseModel):
 
     @classmethod
     def from_neptune_result(cls, data: dict) -> "CatalogueConcept":
+        labels = {}
         descriptions = {}
         identifiers = []
         alternative_labels = []
-        for source_concept in data["source_concepts"]:
-            properties = source_concept["~properties"]
-            source = properties["source"]
 
-            descriptions[source] = properties.get("description")
+        labels["label-derived"] = data["concept"]["~properties"].get("label", "")
 
+        # For now, only extract labels and alternative labels only from source concepts which are explicitly linked
+        # to the concept via HAS_SOURCE_CONCEPT edges
+        for source_concept in data["linked_source_concepts"]:
+            properties = source_concept["~properties"]
+            source = properties["source"]
+            
+            labels[source] = properties.get("label")
             identifiers.append(
                 CatalogueConceptIdentifier(
                     value=properties["id"],
@@ -46,10 +51,16 @@ def from_neptune_result(cls, data: dict) -> "CatalogueConcept":
                 if len(label) > 0:
                     alternative_labels.append(label)
 
+        # Extract descriptions from _all_ source concepts (utilising both HAS_SOURCE_CONCEPT and SAME_AS edges)
+        for source_concept in data["source_concepts"]:
+            properties = source_concept["~properties"]
+            source = properties["source"]
+            descriptions[source] = properties.get("description")
+                
         return CatalogueConcept(
             id=data["concept"]["~properties"]["id"],
-            label=data["concept"]["~properties"].get("label", ""),
             type=data["concept"]["~properties"]["type"],
+            label=get_priority_source_concept_value(labels),
             alternativeLabels=alternative_labels,
             description=get_priority_source_concept_value(descriptions),
             identifiers=identifiers,

catalogue_graph/tests/models/test_catalogue_concept.py

@@ -11,6 +11,26 @@ def test_catalogue_concept_from_neptune_result() -> None:
                     "source": "lc-subjects",
                     "alternative_labels": "alternativeLabels||moreAlternativeLabels",
                     "description": "description",
+                    "label": "Priority label"                    
+                }
+            },
+            {
+                "~properties": {
+                    "id": "id",
+                    "source": "wikidata",
+                    "alternative_labels": "invisibleAlternativeLabel",
+                    "description": "Non-priority description",
+                }
+            }
+        ],
+        "linked_source_concepts": [
+            {
+                "~properties": {
+                    "id": "id",
+                    "source": "lc-subjects",
+                    "alternative_labels": "alternativeLabels||moreAlternativeLabels",
+                    "description": "description",
+                    "label": "Priority label"                    
                 }
             }
         ],
@@ -21,7 +41,7 @@ def test_catalogue_concept_from_neptune_result() -> None:
         identifiers=[
             CatalogueConceptIdentifier(value="id", identifierType="lc-subjects")
         ],
-        label="label",
+        label="Priority label",
         alternativeLabels=["alternativeLabels", "moreAlternativeLabels"],
         description="description",
         type="type",
@@ -46,6 +66,15 @@ def test_catalogue_concept_from_neptune_result_without_alternative_labels() -> N
                 }
             }
         ],
+        "linked_source_concepts": [
+            {
+                "~properties": {
+                    "id": "id",
+                    "source": "nlm-mesh",
+                    "description": "description",
+                }
+            }
+        ],        
     }
 
     assert CatalogueConcept.from_neptune_result(neptune_result) == CatalogueConcept(

catalogue_graph/tests/test_ingestor_loader.py

@@ -41,6 +41,16 @@ def build_test_matrix() -> list[tuple]:
                                 }
                             }
                         ],
+                        "linked_source_concepts": [
+                            {
+                                "~properties": {
+                                    "id": "456",
+                                    "source": "lc-names",
+                                    "alternative_labels": "alternative_label||another_alternative_label",
+                                    "description": "description",
+                                }
+                            }
+                        ],
                     }
                 ]
             },
@@ -92,6 +102,15 @@ def build_test_matrix() -> list[tuple]:
                                 }
                             }
                         ],
+                        "linked_source_concepts": [
+                            {
+                                "~properties": {
+                                    "id": "456",
+                                    "source": "lc-names",
+                                    "description": "description",
+                                }
+                            }
+                        ],                        
                     }
                 ]
             },