Added preprocessing

Author: ZappaBoy

.env.example

@@ -1 +1,2 @@
-OPENAI_API_KEY=sk-<your-api-key>
+OPENAI_API_KEY=sk-<your-api-key>
+HF_TOKEN=your-token

inference.py

@@ -33,8 +33,11 @@
     model = AdvancedTokenAnonymizerModel(best_model)
     for index, row in df.iterrows():
         text = row["source_text"]
+        mask = row["privacy_mask"]
 
-        privacy_mask = ast.literal_eval(row["privacy_mask"]) if str(row['privacy_mask']) != 'nan' else []
+        if type(mask) == str and str(mask) != 'nan':
+            mask = ast.literal_eval(row["privacy_mask"])
+        privacy_mask = mask
 
         trues.append(convert_chars_to_tokens_mapping(text, privacy_mask, model.tokenizer))
         preds.append(model.predict(text))

src/anonymizer/AdvancedTokenAnonymizerModel.py

@@ -1,4 +1,4 @@
-from TokenAnonymizerModel import TokenAnonymizerModel
+from .TokenAnonymizerModel import TokenAnonymizerModel
 
 class AdvancedTokenAnonymizerModel(TokenAnonymizerModel):
     def __init__(self, model_name):