#3544 management script for previous commit

classification/management/commands/remove_invisible_characters.py

@@ -0,0 +1,39 @@
+import json
+
+from django.core.management.base import BaseCommand
+import re
+from classification.models import Classification
+from library.guardian_utils import admin_bot
+
+
+def has_invisible_characters(text):
+    pattern = re.compile(r'[^\u0000-\u007F]')
+    return re.search(pattern, text)
+
+
+def ensure_string(data):
+    if isinstance(data, (dict, list)):
+        return json.dumps(data)
+    elif isinstance(data, str):
+        return data
+    else:
+        return str(data)
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        classifications = Classification.objects.all()
+        user = admin_bot()
+
+        for classification in classifications:
+            evidence = classification.evidence
+
+            for key, value in evidence.items():
+                for k, v in value.items():
+                    if match := has_invisible_characters(ensure_string(v)):
+                        print(f"match found in {classification.id} record, in: {key}")
+                        classification.revalidate(user=user)
+
+        self.stdout.write(
+            self.style.SUCCESS('Invisible characters removed from evidence field for all classifications.'))

classification/migrations/0115_oneoff_remove_non_ascii_characters.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+from manual.operations.manual_operations import ManualOperation
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('classification', '0114_classification_withdraw_reason'),
+    ]
+
+    operations = [
+        ManualOperation(task_id=ManualOperation.task_id_manage(["remove_invisible_characters"]))
+    ]