Clean link references in Telegram crawler

digitalmethodsinitiative · Sep 26, 2024 · cb4b770 · cb4b770
1 parent bfaf23b
commit cb4b770
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/datasources/telegram/search_telegram.py b/datasources/telegram/search_telegram.py
@@ -484,7 +484,7 @@ async def gather_posts(self, client, queries, max_items, min_date, max_date):
                                         # invite links
                                         continue
 
-                                    entity_name = link.split("?")[0].split("#")[0]
+                                    entity_name = link.split("/")[0].split("?")[0].split("#")[0]
                                     linked_entities.add(entity_name)
 
                                 # @references
@@ -493,6 +493,8 @@ async def gather_posts(self, client, queries, max_items, min_date, max_date):
                                     if reference.startswith("@"):
                                         reference = reference[1:]
 
+                                    reference = reference.split("/")[0]
+
                                     linked_entities.add(reference)
 
                             # Check if fwd_from or the resolved entity ID is already queued or has been queried