diff --git a/classification/management/commands/remove_invisible_characters.py b/classification/management/commands/remove_invisible_characters.py index 5c5d64ade..910d1a3ae 100644 --- a/classification/management/commands/remove_invisible_characters.py +++ b/classification/management/commands/remove_invisible_characters.py @@ -7,7 +7,7 @@ def has_invisible_characters(text): - pattern = re.compile(r'[^\u0000-\u007F]') + pattern = "\u00a0" return re.search(pattern, text) diff --git a/classification/models/classification.py b/classification/models/classification.py index 683a51721..a96318cca 100644 --- a/classification/models/classification.py +++ b/classification/models/classification.py @@ -1090,21 +1090,11 @@ def process_entry(self, cell: VCDataCell, source: str): value = cell.value e_key = cell.e_key note = cell.note - # unidecode converts non ascii characters to ascii examples Æ Ö - - def ensure_string(data): - if isinstance(data, (dict, list)): - data = json.dumps(data) - data = unidecode(data) - data = json.loads(data) - return data - elif isinstance(data, str): - return unidecode(data) - else: - return data - value = ensure_string(value) - note = ensure_string(note) + if value and '\u00a0' in value: + value = value.replace('\u00a0', ' ') + if note and '\u00a0' in note: + note = note.replace('\u00a0', ' ') if self.lab.external: cell.validate = False