Filter NUL bytes from TikTok import

digitalmethodsinitiative · Jan 31, 2022 · 564b740 · 564b740
1 parent 3909b58
commit 564b740
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/datasources/tiktok/search_tiktok.py b/datasources/tiktok/search_tiktok.py
@@ -57,7 +57,9 @@ def import_from_file(self, path):
                 if self.interrupted:
                     raise WorkerInterruptedException()
 
-                post = json.loads(line)["data"]
+                # remove NUL bytes here because they trip up a lot of other
+                # things
+                post = json.loads(line.replace("\0", ""))["data"]
 
                 hashtags = [extra["hashtagName"] for extra in post.get("textExtra", []) if "hashtagName" in extra and extra["hashtagName"]]