CU-869805t7e alt names fixes (#520)

* CU-869805t7e: Move getting of applicable category name to the config

* CU-869805t7e: Use alternative category names in eval method

* CU-869805t7e: Reduce indentation

* CU-869805t7e: Reduce indentation (again)

* CU-869805t7e: Some comment fixing due to rearrangements before

* CU-869805t7e: Fix usage of matched class name when encoding category values

* CU-869805t7e: Avoid duplicating exception message

Author: mart-r

medcat/config_meta_cat.py

@@ -1,7 +1,12 @@
+import logging
 from typing import Dict, Any, List
+from collections.abc import Container
 from medcat.config import MixingConfig, BaseModel, Optional
 
 
+logger = logging.getLogger(__name__)
+
+
 class General(MixingConfig, BaseModel):
     """The General part of the MetaCAT config"""
     device: str = 'cpu'
@@ -78,6 +83,18 @@ class General(MixingConfig, BaseModel):
     """If set, the spacy span group that the metacat model will assign annotations.
     Otherwise defaults to doc._.ents or doc.ents per the annotate_overlapping settings"""
 
+    def get_applicable_category_name(self, available_names: Container[str]) -> Optional[str]:
+        if self.category_name in available_names:
+            return self.category_name
+        matches = [cat for cat in self.alternative_category_names if cat in available_names]
+        if len(matches) > 0:
+            logger.info("The category name provided in the config - '%s' is not present in the data. "
+                        "However, the corresponding name - '%s' from the category_name_mapping has been found. "
+                        "Updating the category name...", self.category_name, *matches)
+            self.category_name = matches[0]
+            return self.category_name
+        return None
+
     class Config:
         extra = 'allow'
         validate_assignment = True

medcat/meta_cat.py

@@ -243,18 +243,13 @@ def train_raw(self, data_loaded: Dict, save_dir_path: Optional[str] = None, data
                                  lowercase=g_config['lowercase'])
 
         # Check is the name present
-        category_name = g_config['category_name']
-        category_name_options = g_config['alternative_category_names']
-        if category_name not in data:
-            category_matching = [cat for cat in category_name_options if cat in data.keys()]
-            if len(category_matching) > 0:
-                logger.info("The category name provided in the config - '%s' is not present in the data. However, the corresponding name - '%s' from the category_name_mapping has been found. Updating the category name...",category_name,*category_matching)
-                g_config['category_name'] = category_matching[0]
-                category_name = g_config['category_name']
-            else:
-                raise Exception(
-                    "The category name does not exist in this json file. You've provided '{}', while the possible options are: {}. Additionally, ensure the populate the 'alternative_category_names' attribute to accommodate for variations.".format(
-                        category_name, " | ".join(list(data.keys()))))
+        category_name = g_config.get_applicable_category_name(data)
+        if category_name is None:
+            raise Exception(
+                "The category name does not exist in this json file. You've provided '{}', "
+                "while the possible options are: {}. Additionally, ensure the populate the "
+                "'alternative_category_names' attribute to accommodate for variations.".format(
+                    category_name, " | ".join(list(data.keys()))))
 
         data = data[category_name]
         if data_oversampled:
@@ -344,8 +339,8 @@ def eval(self, json_path: str) -> Dict:
                                  lowercase=g_config['lowercase'])
 
         # Check is the name there
-        category_name = g_config['category_name']
-        if category_name not in data:
+        category_name = g_config.get_applicable_category_name(data)
+        if category_name is None:
             raise Exception("The category name does not exist in this json file.")
 
         data = data[category_name]

medcat/utils/meta_cat/data_utils.py

@@ -188,33 +188,39 @@ def encode_category_values(data: Dict, existing_category_value2id: Optional[Dict
     category_values = set([x[2] for x in data])
 
     # If categoryvalue2id is pre-defined, then making sure it is same as the labels found in the data
-    if len(category_value2id) != 0:
-        if set(category_value2id.keys()) != category_values:
-            # if categoryvalue2id doesn't match the labels in the data, then 'alternative_class_names' has to be defined to check for variations
-            if len(alternative_class_names) != 0:
-                updated_category_value2id = {}
-                for _class in category_value2id.keys():
-                    if _class in category_values:
-                        updated_category_value2id[_class] = category_value2id[_class]
-                    else:
-                        found_in = [sub_map for sub_map in alternative_class_names if _class in sub_map]
-                        if len(found_in) != 0:
-                            class_name_matched = [label for label in found_in[0] if label in category_values]
-                            if len(class_name_matched) != 0:
-                                updated_category_value2id[class_name_matched] = category_value2id[_class]
-                                logger.info("Class name '%s' does not exist in the data; however a variation of it '%s' is present; updating it...",_class,class_name_matched)
-                            else:
-                                raise Exception(
-                                    f"The classes set in the config are not the same as the one found in the data. The classes present in the config vs the ones found in the data - {set(category_value2id.keys())}, {category_values}. Additionally, ensure the populate the 'alternative_class_names' attribute to accommodate for variations.")
-                        else:
-                            raise Exception(f"The classes set in the config are not the same as the one found in the data. The classes present in the config vs the ones found in the data - {set(category_value2id.keys())}, {category_values}. Additionally, ensure the populate the 'alternative_class_names' attribute to accommodate for variations.")
-                category_value2id = copy.deepcopy(updated_category_value2id)
-                logger.info("Updated categoryvalue2id mapping - %s", category_value2id)
-
-            # Else throw an exception since the labels don't match
+    if len(category_value2id) != 0 and set(category_value2id.keys()) != category_values:
+        # if categoryvalue2id doesn't match the labels in the data, then 'alternative_class_names' has to be defined to check for variations
+        if len(alternative_class_names) == 0:
+            # Raise an exception since the labels don't match
+            raise Exception(
+                "The classes set in the config are not the same as the one found in the data. "
+                "The classes present in the config vs the ones found in the data - "
+                f"{set(category_value2id.keys())}, {category_values}. Additionally, ensure the populate the "
+                "'alternative_class_names' attribute to accommodate for variations.")
+        updated_category_value2id = {}
+        for _class in category_value2id.keys():
+            if _class in category_values:
+                updated_category_value2id[_class] = category_value2id[_class]
             else:
-                raise Exception(
-                    f"The classes set in the config are not the same as the one found in the data. The classes present in the config vs the ones found in the data - {set(category_value2id.keys())}, {category_values}. Additionally, ensure the populate the 'alternative_class_names' attribute to accommodate for variations.")
+                found_in = [sub_map for sub_map in alternative_class_names if _class in sub_map]
+                failed_to_find = False
+                if len(found_in) != 0:
+                    class_name_matched = [label for label in found_in[0] if label in category_values]
+                    if len(class_name_matched) != 0:
+                        updated_category_value2id[class_name_matched[0]] = category_value2id[_class]
+                        logger.info("Class name '%s' does not exist in the data; however a variation of it "
+                                    "'%s' is present; updating it...", _class, class_name_matched[0])
+                    else:
+                        failed_to_find = True
+                else:
+                    failed_to_find = True
+                if failed_to_find:
+                    raise Exception("The classes set in the config are not the same as the one found in the data. "
+                                    "The classes present in the config vs the ones found in the data - "
+                                    f"{set(category_value2id.keys())}, {category_values}. Additionally, ensure the "
+                                    "populate the 'alternative_class_names' attribute to accommodate for variations.")
+        category_value2id = copy.deepcopy(updated_category_value2id)
+        logger.info("Updated categoryvalue2id mapping - %s", category_value2id)
 
     # Else create the mapping from the labels found in the data
     else: