data_import: Ignore Rocket.Chat livechat streams/messages.

This resolves the issues reported in zulip#20108, major chunk of which were
due to the incomplete support for importing the livechat streams/messages
in the tool. So, it's best not to import any livechat streams/messages for
now until a complete support for importing the same is developed.

Author: garg3133

zerver/data_import/rocketchat.py

@@ -137,10 +137,6 @@ def process_users(
 def get_stream_name(rc_channel: Dict[str, Any]) -> str:
     if rc_channel.get("teamMain"):
         return f'[TEAM] {rc_channel["name"]}'
-    elif rc_channel.get("t") == "l":
-        # Rocket.Chat livechat channels uses the `fname` field
-        # to specify the channel name.
-        return rc_channel["fname"]
     else:
         return rc_channel["name"]
 
@@ -774,13 +770,15 @@ def map_upload_id_to_upload_data(
     return upload_id_to_upload_data_map
 
 
-def separate_channel_and_private_messages(
+def separate_channel_private_and_livechat_messages(
     messages: List[Dict[str, Any]],
     dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
     direct_id_to_direct_map: Dict[str, Dict[str, Any]],
     huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
+    livechat_id_to_livechat_map: Dict[str, Dict[str, Any]],
     channel_messages: List[Dict[str, Any]],
     private_messages: List[Dict[str, Any]],
+    livechat_messages: List[Dict[str, Any]],
 ) -> None:
     private_channels_list = list(direct_id_to_direct_map.keys()) + list(
         huddle_id_to_huddle_map.keys()
@@ -799,6 +797,8 @@ def separate_channel_and_private_messages(
                 message["rid"] = parent_channel_id
         if message["rid"] in private_channels_list:
             private_messages.append(message)
+        elif message["rid"] in livechat_id_to_livechat_map:
+            livechat_messages.append(message)
         else:
             channel_messages.append(message)
 
@@ -826,6 +826,7 @@ def categorize_channels_and_map_with_id(
     dsc_id_to_dsc_map: Dict[str, Dict[str, Any]],
     direct_id_to_direct_map: Dict[str, Dict[str, Any]],
     huddle_id_to_huddle_map: Dict[str, Dict[str, Any]],
+    livechat_id_to_livechat_map: Dict[str, Dict[str, Any]],
 ) -> None:
     for channel in channel_data:
         if channel.get("prid"):
@@ -835,6 +836,8 @@ def categorize_channels_and_map_with_id(
                 huddle_id_to_huddle_map[channel["_id"]] = channel
             else:
                 direct_id_to_direct_map[channel["_id"]] = channel
+        elif channel["t"] == "l":
+            livechat_id_to_livechat_map[channel["_id"]] = channel
         else:
             room_id_to_room_map[channel["_id"]] = channel
             if channel.get("teamMain"):
@@ -959,6 +962,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
     dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
     direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
     huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+    livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
     categorize_channels_and_map_with_id(
         channel_data=rocketchat_data["room"],
@@ -967,6 +971,7 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
         dsc_id_to_dsc_map=dsc_id_to_dsc_map,
         direct_id_to_direct_map=direct_id_to_direct_map,
         huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+        livechat_id_to_livechat_map=livechat_id_to_livechat_map,
     )
 
     zerver_stream = convert_channel_data(
@@ -1047,14 +1052,17 @@ def do_convert_data(rocketchat_data_dir: str, output_dir: str) -> None:
 
     channel_messages: List[Dict[str, Any]] = []
     private_messages: List[Dict[str, Any]] = []
+    livechat_messages: List[Dict[str, Any]] = []
 
-    separate_channel_and_private_messages(
+    separate_channel_private_and_livechat_messages(
         messages=rocketchat_data["message"],
         dsc_id_to_dsc_map=dsc_id_to_dsc_map,
         direct_id_to_direct_map=direct_id_to_direct_map,
         huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+        livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         channel_messages=channel_messages,
         private_messages=private_messages,
+        livechat_messages=livechat_messages,
     )
 
     total_reactions: List[ZerverFieldsT] = []

zerver/tests/fixtures/rocketchat_fixtures/rocketchat_message.bson

@@ -20,7 +20,7 @@
     process_message_attachment,
     process_users,
     rocketchat_data_to_dict,
-    separate_channel_and_private_messages,
+    separate_channel_private_and_livechat_messages,
 )
 from zerver.data_import.sequencer import IdMapper
 from zerver.data_import.user_handler import UserHandler
@@ -40,11 +40,11 @@ def test_rocketchat_data_to_dict(self) -> None:
         self.assertEqual(rocketchat_data["user"][2]["username"], "harry.potter")
         self.assert_length(rocketchat_data["user"][2]["__rooms"], 10)
 
-        self.assert_length(rocketchat_data["room"], 14)
+        self.assert_length(rocketchat_data["room"], 16)
         self.assertEqual(rocketchat_data["room"][0]["_id"], "GENERAL")
         self.assertEqual(rocketchat_data["room"][0]["name"], "general")
 
-        self.assert_length(rocketchat_data["message"], 73)
+        self.assert_length(rocketchat_data["message"], 87)
         self.assertEqual(rocketchat_data["message"][1]["msg"], "Hey everyone, how's it going??")
         self.assertEqual(rocketchat_data["message"][1]["rid"], "GENERAL")
         self.assertEqual(rocketchat_data["message"][1]["u"]["username"], "priyansh3133")
@@ -186,6 +186,7 @@ def test_categorize_channels_and_map_with_id(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -194,15 +195,17 @@ def test_categorize_channels_and_map_with_id(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
-        self.assert_length(rocketchat_data["room"], 14)
+        self.assert_length(rocketchat_data["room"], 16)
         # Teams are a subset of rooms.
         self.assert_length(room_id_to_room_map, 6)
         self.assert_length(team_id_to_team_map, 1)
         self.assert_length(dsc_id_to_dsc_map, 5)
         self.assert_length(direct_id_to_direct_map, 2)
         self.assert_length(huddle_id_to_huddle_map, 1)
+        self.assert_length(livechat_id_to_livechat_map, 2)
 
         room_id = rocketchat_data["room"][0]["_id"]
         self.assertIn(room_id, room_id_to_room_map)
@@ -224,6 +227,10 @@ def test_categorize_channels_and_map_with_id(self) -> None:
         self.assertIn(huddle_id, huddle_id_to_huddle_map)
         self.assertEqual(huddle_id_to_huddle_map[huddle_id], rocketchat_data["room"][12])
 
+        livechat_id = rocketchat_data["room"][14]["_id"]
+        self.assertIn(livechat_id, livechat_id_to_livechat_map)
+        self.assertEqual(livechat_id_to_livechat_map[livechat_id], rocketchat_data["room"][14])
+
     def test_convert_channel_data(self) -> None:
         fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
         rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@@ -236,6 +243,7 @@ def test_convert_channel_data(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -244,16 +252,9 @@ def test_convert_channel_data(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
-        # Add a dummy livechat channel
-        room_id_to_room_map["2t6Lyzd2KAD3nS8Ch"] = {
-            "_id": "2t6Lyzd2KAD3nS8Ch",
-            "fname": "guest",
-            "t": "l",
-            "ts": datetime.datetime(2019, 11, 6, 0, 38, 42, 796000),
-        }
-
         zerver_stream = convert_channel_data(
             room_id_to_room_map=room_id_to_room_map,
             team_id_to_team_map=team_id_to_team_map,
@@ -262,8 +263,8 @@ def test_convert_channel_data(self) -> None:
         )
 
         # Only rooms are converted to streams.
-        self.assert_length(room_id_to_room_map, 7)
-        self.assert_length(zerver_stream, 7)
+        self.assert_length(room_id_to_room_map, 6)
+        self.assert_length(zerver_stream, 6)
 
         # Normal public stream
         self.assertEqual(zerver_stream[0]["name"], "general")
@@ -299,14 +300,6 @@ def test_convert_channel_data(self) -> None:
         self.assertEqual(zerver_stream[5]["stream_post_policy"], 1)
         self.assertEqual(zerver_stream[5]["realm"], realm_id)
 
-        # Livechat channel
-        self.assertEqual(zerver_stream[6]["name"], "guest")
-        self.assertEqual(zerver_stream[6]["invite_only"], False)
-        self.assertEqual(zerver_stream[6]["description"], "")
-        self.assertEqual(zerver_stream[6]["rendered_description"], "")
-        self.assertEqual(zerver_stream[6]["stream_post_policy"], 1)
-        self.assertEqual(zerver_stream[6]["realm"], realm_id)
-
     def test_convert_stream_subscription_data(self) -> None:
         fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
         rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
@@ -334,6 +327,7 @@ def test_convert_stream_subscription_data(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -342,6 +336,7 @@ def test_convert_stream_subscription_data(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
         zerver_stream = convert_channel_data(
@@ -438,6 +433,7 @@ def test_convert_huddle_data(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -446,6 +442,7 @@ def test_convert_huddle_data(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
         zerver_huddle = convert_huddle_data(
@@ -536,6 +533,7 @@ def test_map_receiver_id_to_recipient_id(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -544,6 +542,7 @@ def test_map_receiver_id_to_recipient_id(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
         zerver_stream = convert_channel_data(
@@ -604,7 +603,7 @@ def test_map_receiver_id_to_recipient_id(self) -> None:
             huddle_id_to_recipient_id[zerver_recipient[12]["type_id"]], zerver_recipient[12]["id"]
         )
 
-    def test_separate_channel_and_private_messages(self) -> None:
+    def test_separate_channel_private_and_livechat_messages(self) -> None:
         fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")
         rocketchat_data = rocketchat_data_to_dict(fixture_dir_name)
 
@@ -613,6 +612,7 @@ def test_separate_channel_and_private_messages(self) -> None:
         dsc_id_to_dsc_map: Dict[str, Dict[str, Any]] = {}
         direct_id_to_direct_map: Dict[str, Dict[str, Any]] = {}
         huddle_id_to_huddle_map: Dict[str, Dict[str, Any]] = {}
+        livechat_id_to_livechat_map: Dict[str, Dict[str, Any]] = {}
 
         categorize_channels_and_map_with_id(
             channel_data=rocketchat_data["room"],
@@ -621,23 +621,28 @@ def test_separate_channel_and_private_messages(self) -> None:
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
         )
 
         channel_messages: List[Dict[str, Any]] = []
         private_messages: List[Dict[str, Any]] = []
+        livechat_messages: List[Dict[str, Any]] = []
 
-        separate_channel_and_private_messages(
+        separate_channel_private_and_livechat_messages(
             messages=rocketchat_data["message"],
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
             channel_messages=channel_messages,
             private_messages=private_messages,
+            livechat_messages=livechat_messages,
         )
 
-        self.assert_length(rocketchat_data["message"], 73)
-        self.assert_length(channel_messages, 62)
+        self.assert_length(rocketchat_data["message"], 87)
+        self.assert_length(channel_messages, 68)
         self.assert_length(private_messages, 11)
+        self.assert_length(livechat_messages, 8)
 
         self.assertIn(rocketchat_data["message"][0], channel_messages)
         self.assertIn(rocketchat_data["message"][1], channel_messages)
@@ -647,6 +652,10 @@ def test_separate_channel_and_private_messages(self) -> None:
         self.assertIn(rocketchat_data["message"][12], private_messages)
         self.assertIn(rocketchat_data["message"][50], private_messages)  # Huddle message
 
+        self.assertIn(rocketchat_data["message"][79], livechat_messages)
+        self.assertIn(rocketchat_data["message"][83], livechat_messages)
+        self.assertIn(rocketchat_data["message"][86], livechat_messages)
+
         # Message in a Discussion originating from a direct channel
         self.assertIn(rocketchat_data["message"][70], private_messages)
         self.assertIn(rocketchat_data["message"][70]["rid"], direct_id_to_direct_map)
@@ -677,19 +686,23 @@ def test_separate_channel_and_private_messages(self) -> None:
 
         channel_messages = []
         private_messages = []
+        livechat_messages = []
 
-        separate_channel_and_private_messages(
+        separate_channel_private_and_livechat_messages(
             messages=rocketchat_data["message"],
             dsc_id_to_dsc_map=dsc_id_to_dsc_map,
             direct_id_to_direct_map=direct_id_to_direct_map,
             huddle_id_to_huddle_map=huddle_id_to_huddle_map,
+            livechat_id_to_livechat_map=livechat_id_to_livechat_map,
             channel_messages=channel_messages,
             private_messages=private_messages,
+            livechat_messages=livechat_messages,
         )
 
-        # No new message added to channel or private messages
-        self.assert_length(channel_messages, 62)
+        # No new message added to channel, private or livechat messages
+        self.assert_length(channel_messages, 68)
         self.assert_length(private_messages, 11)
+        self.assert_length(livechat_messages, 8)
 
     def test_map_upload_id_to_upload_data(self) -> None:
         fixture_dir_name = self.fixture_file_name("", "rocketchat_fixtures")