Migrate to prom-ex metrics (#1198)

* Update grafana dashboard

* feat: add prom_ex dependency and update required dependencies

had to update plug and finch to update telemetry_poller for prom_ex

* feat: add prom ex module to gather metrics from beam

* feat: add basic prom ex code and plugin

* replicate telemetry module events using prom ex

* remove duplicate bot registry

* remove telemetry module

* restore vm metrics event

they are already being generated by telemetry_poller, and we were
registering them before so I'll keep these so as not to break
pre-existing grafana panels

* use pre-existing metrics_endpoint_port

* remove deleted module from application

* fix: use underscore for big numbers

* run formatter

* Fix typo

Co-authored-by: Nicolás Sanchez <[email protected]>

* add missing bots count metric

---------

Co-authored-by: Nicolas Sanchez <[email protected]>

Author: mFragaBA

apps/arena/lib/arena/application.ex

@@ -24,10 +24,10 @@ defmodule Arena.Application do
       Arena.Authentication.GatewaySigner,
       Arena.Bots.BotSupervisor,
       Arena.Bots.PathfindingGrid,
+      Arena.PromEx,
       # Start a worker by calling: Arena.Worker.start_link(arg)
       # {Arena.Worker, arg},
       # Start to serve requests, typically the last entry
-      ArenaWeb.Telemetry,
       ArenaWeb.Endpoint
     ]
 

apps/arena/lib/arena/prom_ex.ex

@@ -0,0 +1,14 @@
+defmodule Arena.PromEx do
+  @moduledoc """
+  This module integrates the PromEx library. It sets up PromEx plugins and pre-built dashboards for the node.
+  """
+  use PromEx, otp_app: :arena
+
+  @impl true
+  def plugins() do
+    [
+      PromEx.Plugins.Beam,
+      Arena.PromExPlugin
+    ]
+  end
+end

apps/arena/lib/arena/prom_ex_plugin.ex

@@ -0,0 +1,56 @@
+defmodule Arena.PromExPlugin do
+  @moduledoc """
+  This module defines our custom PromEx plugin.
+  It contains all our custom metrics that are displayed on the grafana dashboard.
+  """
+
+  use PromEx.Plugin
+
+  @impl true
+  def event_metrics(_opts) do
+    [
+      Event.build(:game_metrics, [
+        sum("arena.game.count", description: "Number of games in progress"),
+        ## TODO: this metric is an attempt to gather data to properly set the buckets for the distribution metric below
+        last_value("arena.game.tick.duration_measure",
+          description: "Last game tick duration",
+          unit: {:native, :nanosecond}
+        ),
+        ## TODO: Buckets probably need to be redefined, currently they all fall under the first bucket
+        distribution("arena.game.tick.duration",
+          description: "Time spent on running a game tick",
+          unit: {:native, :nanosecond},
+          reporter_options: [buckets: [7_500_000.0, 15_000_000.0, 22_500_000.0]]
+        ),
+        sum("arena.clients.count", description: "Number of clients (websockets) connected"),
+        sum([:bots, :count], description: "Amount of active bots")
+      ]),
+      Event.build(:vm_metrics, [
+        last_value("vm.memory.total", unit: {:byte, :kilobyte}),
+        last_value("vm.total_run_queue_lengths.total", description: "Total run queue length of CPU and IO schedulers"),
+        last_value("vm.total_run_queue_lengths.cpu", description: "Run queue length of CPU scheduler"),
+        last_value("vm.total_run_queue_lengths.io", description: "Run queue length of IO scheduler")
+      ])
+    ]
+  end
+
+  @impl true
+  def polling_metrics(_opts) do
+    poll_rate = 10_000
+
+    [
+      Polling.build(:periodic_measurements, poll_rate, {Arena.Utils, :message_queue_lengths, []}, [
+        # All BEAM processes message queues
+        last_value([:vm, :message_queue, :length], tags: [:process]),
+        # Bots
+        distribution([:bots, :message_queue, :length],
+          reporter_options: [buckets: [0, 10, 100, 1_000, 10_000, 100_000]]
+        ),
+        # GameUpdater
+        distribution([:game_updater, :message_queue, :length],
+          reporter_options: [buckets: [0, 10, 100, 1_000, 10_000, 100_000]]
+        )
+      ])
+    ]
+  end
+end

apps/arena/lib/arena_web/telemetry.ex

@@ -55,7 +55,8 @@ defmodule Arena.MixProject do
       {:plug_cowboy, "~> 2.5"},
       {:toxiproxy_ex, "~> 1.1.1"},
       {:joken, "~> 2.6"},
-      {:new_relic_agent, "~> 1.0", only: [:prod]}
+      {:new_relic_agent, "~> 1.0", only: [:prod]},
+      {:prom_ex, "~> 1.11.0"}
     ]
   end
 

apps/arena/mix.exs

@@ -58,6 +58,12 @@ metrics_endpoint_port =
 config :arena, :gateway_url, System.get_env("GATEWAY_URL") || "http://localhost:4001"
 config :arena, :metrics_endpoint_port, metrics_endpoint_port
 
+config :arena, Arena.PromEx,
+  metrics_server: [
+    port: metrics_endpoint_port,
+    auth_strategy: :none
+  ]
+
 if System.get_env("PHX_SERVER") do
   config :arena, ArenaWeb.Endpoint, server: true
 end