Merge branch 'main' into plando_items_rewrite

BaseClasses.py

@@ -887,21 +887,40 @@ def sweep_for_advancements(self, locations: Optional[Iterable[Location]] = None)
     def has(self, item: str, player: int, count: int = 1) -> bool:
         return self.prog_items[player][item] >= count
 
+    # for loops are specifically used in all/any/count methods, instead of all()/any()/sum(), to avoid the overhead of
+    # creating and iterating generator instances. In `return all(player_prog_items[item] for item in items)`, the
+    # argument to all() would be a new generator instance, for example.
     def has_all(self, items: Iterable[str], player: int) -> bool:
         """Returns True if each item name of items is in state at least once."""
-        return all(self.prog_items[player][item] for item in items)
+        player_prog_items = self.prog_items[player]
+        for item in items:
+            if not player_prog_items[item]:
+                return False
+        return True
 
     def has_any(self, items: Iterable[str], player: int) -> bool:
         """Returns True if at least one item name of items is in state at least once."""
-        return any(self.prog_items[player][item] for item in items)
+        player_prog_items = self.prog_items[player]
+        for item in items:
+            if player_prog_items[item]:
+                return True
+        return False
 
     def has_all_counts(self, item_counts: Mapping[str, int], player: int) -> bool:
         """Returns True if each item name is in the state at least as many times as specified."""
-        return all(self.prog_items[player][item] >= count for item, count in item_counts.items())
+        player_prog_items = self.prog_items[player]
+        for item, count in item_counts.items():
+            if player_prog_items[item] < count:
+                return False
+        return True
 
     def has_any_count(self, item_counts: Mapping[str, int], player: int) -> bool:
         """Returns True if at least one item name is in the state at least as many times as specified."""
-        return any(self.prog_items[player][item] >= count for item, count in item_counts.items())
+        player_prog_items = self.prog_items[player]
+        for item, count in item_counts.items():
+            if player_prog_items[item] >= count:
+                return True
+        return False
 
     def count(self, item: str, player: int) -> int:
         return self.prog_items[player][item]
@@ -929,11 +948,20 @@ def has_from_list_unique(self, items: Iterable[str], player: int, count: int) ->
 
     def count_from_list(self, items: Iterable[str], player: int) -> int:
         """Returns the cumulative count of items from a list present in state."""
-        return sum(self.prog_items[player][item_name] for item_name in items)
+        player_prog_items = self.prog_items[player]
+        total = 0
+        for item_name in items:
+            total += player_prog_items[item_name]
+        return total
 
     def count_from_list_unique(self, items: Iterable[str], player: int) -> int:
         """Returns the cumulative count of items from a list present in state. Ignores duplicates of the same item."""
-        return sum(self.prog_items[player][item_name] > 0 for item_name in items)
+        player_prog_items = self.prog_items[player]
+        total = 0
+        for item_name in items:
+            if player_prog_items[item_name] > 0:
+                total += 1
+        return total
 
     # item name group related
     def has_group(self, item_name_group: str, player: int, count: int = 1) -> bool:

CommonClient.py

@@ -709,8 +709,16 @@ def handle_connection_loss(self, msg: str) -> None:
         logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True})
         self._messagebox_connection_loss = self.gui_error(msg, exc_info[1])
 
-    def make_gui(self) -> typing.Type["kvui.GameManager"]:
-        """To return the Kivy App class needed for run_gui so it can be overridden before being built"""
+    def make_gui(self) -> "type[kvui.GameManager]":
+        """
+        To return the Kivy `App` class needed for `run_gui` so it can be overridden before being built
+
+        Common changes are changing `base_title` to update the window title of the client and
+        updating `logging_pairs` to automatically make new tabs that can be filled with their respective logger.
+
+        ex. `logging_pairs.append(("Foo", "Bar"))`
+        will add a "Bar" tab which follows the logger returned from `logging.getLogger("Foo")`
+        """
         from kvui import GameManager
 
         class TextManager(GameManager):
@@ -899,6 +907,7 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
             ctx.disconnected_intentionally = True
             ctx.event_invalid_game()
         elif 'IncompatibleVersion' in errors:
+            ctx.disconnected_intentionally = True
             raise Exception('Server reported your client version as incompatible. '
                             'This probably means you have to update.')
         elif 'InvalidItemsHandling' in errors:
@@ -1087,7 +1096,7 @@ async def server_auth(self, password_requested: bool = False):
             if password_requested and not self.password:
                 await super(TextContext, self).server_auth(password_requested)
             await self.get_username()
-            await self.send_connect()
+            await self.send_connect(game="")
 
         def on_package(self, cmd: str, args: dict):
             if cmd == "Connected":

LttPAdjuster.py

@@ -33,10 +33,15 @@
 WINDOW_MIN_WIDTH = 425
 
 class AdjusterWorld(object):
+    class AdjusterSubWorld(object):
+        def __init__(self, random):
+            self.random = random
+
     def __init__(self, sprite_pool):
         import random
         self.sprite_pool = {1: sprite_pool}
         self.per_slot_randoms = {1: random}
+        self.worlds = {1: self.AdjusterSubWorld(random)}
 
 
 class ArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter):

MultiServer.py

@@ -28,9 +28,11 @@
 
 if typing.TYPE_CHECKING:
     import ssl
+    from NetUtils import ServerConnection
 
-import websockets
 import colorama
+import websockets
+from websockets.extensions.permessage_deflate import PerMessageDeflate
 try:
     # ponyorm is a requirement for webhost, not default server, so may not be importable
     from pony.orm.dbapiprovider import OperationalError
@@ -119,13 +121,14 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int:
 
 class Client(Endpoint):
     version = Version(0, 0, 0)
-    tags: typing.List[str] = []
+    tags: typing.List[str]
     remote_items: bool
     remote_start_inventory: bool
     no_items: bool
     no_locations: bool
+    no_text: bool
 
-    def __init__(self, socket: websockets.WebSocketServerProtocol, ctx: Context):
+    def __init__(self, socket: "ServerConnection", ctx: Context) -> None:
         super().__init__(socket)
         self.auth = False
         self.team = None
@@ -175,6 +178,7 @@ class Context:
                       "compatibility": int}
     # team -> slot id -> list of clients authenticated to slot.
     clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]]
+    endpoints: list[Client]
     locations: LocationStore  # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
     location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]]
     hints_used: typing.Dict[typing.Tuple[int, int], int]
@@ -364,18 +368,28 @@ async def broadcast_send_encoded_msgs(self, endpoints: typing.Iterable[Endpoint]
             return True
 
     def broadcast_all(self, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in self.endpoints if endpoint.auth)
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in self.endpoints
+            if endpoint.auth and not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast_text_all(self, text: str, additional_arguments: dict = {}):
         self.logger.info("Notice (all): %s" % text)
         self.broadcast_all([{**{"cmd": "PrintJSON", "data": [{ "text": text }]}, **additional_arguments}])
 
     def broadcast_team(self, team: int, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in itertools.chain.from_iterable(self.clients[team].values()))
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in itertools.chain.from_iterable(self.clients[team].values())
+            if not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast(self, endpoints: typing.Iterable[Client], msgs: typing.List[dict]):
         msgs = self.dumper(msgs)
@@ -389,13 +403,13 @@ async def disconnect(self, endpoint: Client):
         await on_client_disconnected(self, endpoint)
 
     def notify_client(self, client: Client, text: str, additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         self.logger.info("Notice (Player %s in team %d): %s" % (client.name, client.team + 1, text))
         async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}]))
 
     def notify_client_multiple(self, client: Client, texts: typing.List[str], additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         async_start(self.send_msgs(client,
                                    [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}
@@ -760,7 +774,7 @@ def notify_hints(self, team: int, hints: typing.List[Hint], only_new: bool = Fal
             self.on_new_hint(team, slot)
         for slot, hint_data in concerns.items():
             if recipients is None or slot in recipients:
-                clients = self.clients[team].get(slot)
+                clients = filter(lambda c: not c.no_text, self.clients[team].get(slot, []))
                 if not clients:
                     continue
                 client_hints = [datum[1] for datum in sorted(hint_data, key=lambda x: x[0].finding_player != slot)]
@@ -819,7 +833,7 @@ def update_aliases(ctx: Context, team: int):
             async_start(ctx.send_encoded_msgs(client, cmd))
 
 
-async def server(websocket, path: str = "/", ctx: Context = None):
+async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = None) -> None:
     client = Client(websocket, ctx)
     ctx.endpoints.append(client)
 
@@ -910,6 +924,10 @@ async def on_client_joined(ctx: Context, client: Client):
                               "If your client supports it, "
                               "you may have additional local commands you can list with /help.",
                       {"type": "Tutorial"})
+    if not any(isinstance(extension, PerMessageDeflate) for extension in client.socket.extensions):
+        ctx.notify_client(client, "Warning: your client does not support compressed websocket connections! "
+                                  "It may stop working in the future. If you are a player, please report this to the "
+                                  "client's developer.")
     ctx.client_connection_timers[client.team, client.slot] = datetime.datetime.now(datetime.timezone.utc)
 
 
@@ -1060,21 +1078,37 @@ def send_items_to(ctx: Context, team: int, target_slot: int, *items: NetworkItem
 
 def register_location_checks(ctx: Context, team: int, slot: int, locations: typing.Iterable[int],
                              count_activity: bool = True):
+    slot_locations = ctx.locations[slot]
     new_locations = set(locations) - ctx.location_checks[team, slot]
-    new_locations.intersection_update(ctx.locations[slot])  # ignore location IDs unknown to this multidata
+    new_locations.intersection_update(slot_locations)  # ignore location IDs unknown to this multidata
     if new_locations:
         if count_activity:
             ctx.client_activity_timers[team, slot] = datetime.datetime.now(datetime.timezone.utc)
+
+        sortable: list[tuple[int, int, int, int]] = []
         for location in new_locations:
-            item_id, target_player, flags = ctx.locations[slot][location]
+            # extract all fields to avoid runtime overhead in LocationStore
+            item_id, target_player, flags = slot_locations[location]
+            # sort/group by receiver and item
+            sortable.append((target_player, item_id, location, flags))
+
+        info_texts: list[dict[str, typing.Any]] = []
+        for target_player, item_id, location, flags in sorted(sortable):
             new_item = NetworkItem(item_id, location, slot, flags)
             send_items_to(ctx, team, target_player, new_item)
 
             ctx.logger.info('(Team #%d) %s sent %s to %s (%s)' % (
                 team + 1, ctx.player_names[(team, slot)], ctx.item_names[ctx.slot_info[target_player].game][item_id],
                 ctx.player_names[(team, target_player)], ctx.location_names[ctx.slot_info[slot].game][location]))
-            info_text = json_format_send_event(new_item, target_player)
-            ctx.broadcast_team(team, [info_text])
+            if len(info_texts) >= 140:
+                # split into chunks that are close to compression window of 64K but not too big on the wire
+                # (roughly 1300-2600 bytes after compression depending on repetitiveness)
+                ctx.broadcast_team(team, info_texts)
+                info_texts.clear()
+            info_texts.append(json_format_send_event(new_item, target_player))
+        ctx.broadcast_team(team, info_texts)
+        del info_texts
+        del sortable
 
         ctx.location_checks[team, slot] |= new_locations
         send_new_items(ctx)
@@ -1787,7 +1821,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             ctx.clients[team][slot].append(client)
             client.version = args['version']
             client.tags = args['tags']
-            client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+            client.no_locations = "TextOnly" in client.tags or "Tracker" in client.tags
+            # set NoText for old PopTracker clients that predate the tag to save traffic
+            client.no_text = "NoText" in client.tags or ("PopTracker" in client.tags and client.version < (0, 5, 1))
             connected_packet = {
                 "cmd": "Connected",
                 "team": client.team, "slot": client.slot,
@@ -1860,6 +1896,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
                 client.tags = args["tags"]
                 if set(old_tags) != set(client.tags):
                     client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+                    client.no_text = "NoText" in client.tags or (
+                        "PopTracker" in client.tags and client.version < (0, 5, 1)
+                    )
                     ctx.broadcast_text_all(
                         f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) has changed tags "
                         f"from {old_tags} to {client.tags}.",

NetUtils.py

@@ -5,17 +5,18 @@
 import warnings
 from json import JSONEncoder, JSONDecoder
 
-import websockets
+if typing.TYPE_CHECKING:
+    from websockets import WebSocketServerProtocol as ServerConnection
 
 from Utils import ByValue, Version
 
 
 class HintStatus(ByValue, enum.IntEnum):
-    HINT_FOUND = 0
-    HINT_UNSPECIFIED = 1
+    HINT_UNSPECIFIED = 0
     HINT_NO_PRIORITY = 10
     HINT_AVOID = 20
     HINT_PRIORITY = 30
+    HINT_FOUND = 40
 
 
 class JSONMessagePart(typing.TypedDict, total=False):
@@ -151,7 +152,7 @@ def _object_hook(o: typing.Any) -> typing.Any:
 
 
 class Endpoint:
-    socket: websockets.WebSocketServerProtocol
+    socket: "ServerConnection"
 
     def __init__(self, socket):
         self.socket = socket

Options.py

@@ -1710,7 +1710,7 @@ def dump_player_options(multiworld: MultiWorld) -> None:
             }
             output.append(player_output)
             for option_key, option in world.options_dataclass.type_hints.items():
-                if issubclass(Removed, option):
+                if option.visibility == Visibility.none:
                     continue
                 display_name = getattr(option, "display_name", option_key)
                 player_output[display_name] = getattr(world.options, option_key).current_option_name

WebHostLib/static/assets/faq/en.md

@@ -22,7 +22,7 @@ players to rely upon each other to complete their game.
 
 While a multiworld game traditionally requires all players to be playing the same game, a multi-game multiworld allows
 players to randomize any of the supported games, and send items between them. This allows players of different
-games to interact with one another in a single multiplayer environment.  Archipelago supports multi-game multiworld.
+games to interact with one another in a single multiplayer environment.  Archipelago supports multi-game multiworlds.
 Here is a list of our [Supported Games](https://archipelago.gg/games).
 
 ## Can I generate a single-player game with Archipelago?