Shutdown websocket connection after game ended (#1206)

mFragaBA · Nico-Sanchez · web-flow · commit 8a66267411c3 · 2025-05-09T17:53:53.000-03:00
* Timeout ws connections when game ends

* Add os_mon dependency to monitor OS values

* Add new telemetry events for OS memory and cpu metrics

* Update grafana dashboard: add cpu and memory panels

* replace old request code and comment issue with creating bot mid game

* remove extra stuff

* Fix todo comment to restore bots replacing disconnected players

* Add new LOADTEST_ALONE_MODE env-var

---------

Co-authored-by: Nicolas Sanchez &lt;sanchez.nicolas96@gmail.com&gt;
diff --git a/.github/workflows/arena-argentina-testing-deploy.yaml b/.github/workflows/arena-argentina-testing-deploy.yaml
@@ -55,6 +55,7 @@ jobs:
           JWT_PRIVATE_KEY_BASE_64: ${{ secrets.JWT_PRIVATE_KEY_BASE_64 }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
           set -ex
@@ -73,4 +74,5 @@ jobs:
                 JWT_PRIVATE_KEY_BASE_64=${JWT_PRIVATE_KEY_BASE_64} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 /Users/${SSH_USERNAME}/deploy-script/deploy-macos.sh
diff --git a/.github/workflows/arena-chile-testing-deploy.yml b/.github/workflows/arena-chile-testing-deploy.yml
@@ -52,6 +52,7 @@ jobs:
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           XDG_RUNTIME_DIR: ${{ vars.XDG_RUNTIME_DIR }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
@@ -71,6 +72,7 @@ jobs:
                 SECRET_KEY_BASE=${SECRET_KEY_BASE} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/arena-europe-testing-deploy.yml b/.github/workflows/arena-europe-testing-deploy.yml
@@ -55,6 +55,7 @@ jobs:
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -73,5 +74,6 @@ jobs:
                 SECRET_KEY_BASE=${SECRET_KEY_BASE} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/arena-usa-testing-deploy.yml b/.github/workflows/arena-usa-testing-deploy.yml
@@ -55,6 +55,7 @@ jobs:
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -73,5 +74,6 @@ jobs:
                 SECRET_KEY_BASE=${SECRET_KEY_BASE} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/bot-manager-deploy.yml b/.github/workflows/bot-manager-deploy.yml
@@ -53,6 +53,7 @@ jobs:
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -69,5 +70,6 @@ jobs:
                 SECRET_KEY_BASE=${SECRET_KEY_BASE} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/central-europe-staging-deploy.yml b/.github/workflows/central-europe-staging-deploy.yml
@@ -58,6 +58,7 @@ jobs:
           JWT_PRIVATE_KEY_BASE_64: ${{ secrets.JWT_PRIVATE_KEY_BASE_64 }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME_STAGING }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -79,5 +80,6 @@ jobs:
                 JWT_PRIVATE_KEY_BASE_64=${JWT_PRIVATE_KEY_BASE_64} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/central-europe-testing-deploy.yml b/.github/workflows/central-europe-testing-deploy.yml
@@ -58,6 +58,7 @@ jobs:
           JWT_PRIVATE_KEY_BASE_64: ${{ secrets.JWT_PRIVATE_KEY_BASE_64 }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME_TESTING }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -79,5 +80,6 @@ jobs:
                 JWT_PRIVATE_KEY_BASE_64=${JWT_PRIVATE_KEY_BASE_64} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/.github/workflows/deploy-new-arena-server.yml b/.github/workflows/deploy-new-arena-server.yml
@@ -63,12 +63,14 @@ jobs:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
         run: |
           ssh ${SSH_USERNAME}@${SSH_HOST} \
             SSH_APP_USERNAME=${SSH_APP_USERNAME} \
             DATABASE_URL=${DATABASE_URL} \
             SECRET_KEY_BASE=${SECRET_KEY_BASE} \
             NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
             /home/${SSH_USERNAME}/mirra_backend/devops/setup-admin-deps.sh
 
       - name: Tailscale
diff --git a/.github/workflows/loadtest-europe-client-deploy.yml b/.github/workflows/loadtest-europe-client-deploy.yml
@@ -64,6 +64,7 @@ jobs:
           SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }}
           NEWRELIC_APP_NAME: ${{ vars.NEWRELIC_APP_NAME_LOADTEST_CLIENT }}
           NEWRELIC_KEY: ${{ secrets.NEWRELIC_KEY }}
+          LOADTEST_ALONE_MODE: ${{ vars.LOADTEST_ALONE_MODE }}
           DBUS_SESSION_BUS_ADDRESS: ${{ vars.DBUS_SESSION_BUS_ADDRESS }}
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
@@ -84,5 +85,6 @@ jobs:
                 SECRET_KEY_BASE=${SECRET_KEY_BASE} \
                 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME} \
                 NEWRELIC_KEY=${NEWRELIC_KEY} \
+                LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE} \
                 DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS} \
                 /home/${SSH_USERNAME}/deploy-script/deploy.sh
diff --git a/apps/arena/lib/arena/game_socket_handler.ex b/apps/arena/lib/arena/game_socket_handler.ex
@@ -5,7 +5,6 @@ defmodule Arena.GameSocketHandler do
   require Logger
   alias Arena.Authentication.GatewaySigner
   alias Arena.Authentication.GatewayTokenManager
-  alias Arena.Utils
   alias Arena.Serialization
   alias Arena.GameUpdater
   alias Arena.Serialization.GameEvent
@@ -108,6 +107,7 @@ defmodule Arena.GameSocketHandler do
   @impl true
   def websocket_info({:game_finished, game_state}, state) do
     # Logger.info("Websocket info, Message: GAME FINISHED")
+    Process.send_after(self(), :terminate_connection, 10_000)
     {:reply, {:binary, game_state}, state}
   end
 
@@ -172,29 +172,28 @@ defmodule Arena.GameSocketHandler do
     end
   end
 
+  def websocket_info(:terminate_connection, state) do
+    {:stop, state}
+  end
+
   @impl true
   def websocket_info(message, state) do
     Logger.info("You should not be here: #{inspect(message)}")
     {:reply, {:binary, Jason.encode!(%{})}, state}
   end
 
   @impl true
-  def terminate(_reason, _req, %{game_finished: false, player_alive: true} = state) do
+  def terminate(reason, _req, %{game_finished: false, player_alive: true} = state) do
     :telemetry.execute([:arena, :clients], %{count: -1})
+    Logger.error("Player #{state.player_id} terminating: #{inspect(reason)}")
 
-    if Application.get_env(:arena, :spawn_bots) do
-      spawn(fn ->
-        Finch.build(:get, Utils.get_bot_connection_url(state.game_id, state.client_id))
-        |> Finch.request(Arena.Finch)
-      end)
-    end
-
-    :ok
+    # TODO: We have to reconnect the bots here, we lost this behavior when moved the bots inside Arena.
+    # https://github.com/lambdaclass/mirra_backend/issues/1128
   end
 
-  def terminate(_reason, _req, _state) do
+  def terminate(reason, _req, state) do
     :telemetry.execute([:arena, :clients], %{count: -1})
-    :ok
+    Logger.error("Player #{state.player_id} terminating: #{inspect(reason)}")
   end
 
   defp to_broadcast_config(config) do
diff --git a/apps/arena/lib/arena/prom_ex_plugin.ex b/apps/arena/lib/arena/prom_ex_plugin.ex
@@ -39,7 +39,7 @@ defmodule Arena.PromExPlugin do
     poll_rate = 10_000
 
     [
-      Polling.build(:periodic_measurements, poll_rate, {Arena.Utils, :message_queue_lengths, []}, [
+      Polling.build(:periodic_measurements, poll_rate, {Arena.Utils, :periodic_measurements, []}, [
         # All BEAM processes message queues
         last_value([:vm, :message_queue, :length], tags: [:process]),
         # Bots
@@ -49,7 +49,16 @@ defmodule Arena.PromExPlugin do
         # GameUpdater
         distribution([:game_updater, :message_queue, :length],
           reporter_options: [buckets: [0, 10, 100, 1_000, 10_000, 100_000]]
-        )
+        ),
+        # OS
+
+        last_value([:os, :cpu_usage], []),
+        last_value([:os, :system_total_memory], unit: {:byte, :megabyte}),
+        last_value([:os, :free_memory], unit: {:byte, :megabyte}),
+        last_value([:os, :buffered_memory], unit: {:byte, :megabyte}),
+        last_value([:os, :cached_memory], unit: {:byte, :megabyte}),
+        last_value([:os, :total_swap], unit: {:byte, :megabyte}),
+        last_value([:os, :free_swap], unit: {:byte, :megabyte})
       ])
     ]
   end
diff --git a/apps/arena/lib/arena/utils.ex b/apps/arena/lib/arena/utils.ex
@@ -141,10 +141,16 @@ defmodule Arena.Utils do
   # Time to wait to start game with any amount of clients
   def start_timeout_ms(), do: 4_000
 
-  def message_queue_lengths() do
+  def periodic_measurements() do
     register_all_processes_queues()
     register_bots_queues()
     register_game_updater_queues()
+    register_os_usage()
+  end
+
+  def register_os_usage() do
+    :telemetry.execute([:os], %{cpu_usage: :cpu_sup.util()})
+    :telemetry.execute([:os], Map.new(:memsup.get_system_memory_data()))
   end
 
   def register_all_processes_queues() do
diff --git a/apps/arena/mix.exs b/apps/arena/mix.exs
@@ -23,7 +23,7 @@ defmodule Arena.MixProject do
   def application do
     [
       mod: {Arena.Application, []},
-      extra_applications: [:logger, :runtime_tools]
+      extra_applications: [:logger, :runtime_tools, :os_mon]
     ]
   end
 
diff --git a/devops/deploy-macos.sh b/devops/deploy-macos.sh
@@ -51,6 +51,7 @@ LOADTEST_BRAZIL_HOST=${LOADTEST_BRAZIL_HOST}
 LOADTEST_CHILE_HOST=${LOADTEST_CHILE_HOST}
 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME}
 NEWRELIC_KEY=${NEWRELIC_KEY}
+LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE}
 EOF
 
 export PHX_HOST=$PHX_HOST
@@ -77,6 +78,7 @@ export LOADTEST_BRAZIL_HOST=$LOADTEST_BRAZIL_HOST
 export LOADTEST_CHILE_HOST=$LOADTEST_CHILE_HOST
 export NEWRELIC_APP_NAME=$NEWRELIC_APP_NAME
 export NEWRELIC_KEY=$NEWRELIC_KEY
+export LOADTEST_ALONE_MODE=$LOADTEST_ALONE_MODE
 
 /Users/lambdaclass/arena/mirra_backend/_build/prod/rel/arena/bin/arena stop
 
diff --git a/devops/deploy.sh b/devops/deploy.sh
@@ -73,6 +73,7 @@ LOADTEST_BRAZIL_HOST=${LOADTEST_BRAZIL_HOST}
 LOADTEST_CHILE_HOST=${LOADTEST_CHILE_HOST}
 NEWRELIC_APP_NAME=${NEWRELIC_APP_NAME}
 NEWRELIC_KEY=${NEWRELIC_KEY}
+LOADTEST_ALONE_MODE=${LOADTEST_ALONE_MODE}
 EOF
 
 systemctl --user stop ${RELEASE}${_SERVICE_SUFFIX}
diff --git a/grafana_arena_dashboard.json b/grafana_arena_dashboard.json
