Treat gateways as Nym Nodes (#5504)

* Generate GW moniker if missing

Beside that:
- clear up gw nomenclature
- adjust counting when legacy nodes are present in nym node APIs
- create utils module

* Store gatewy descriptions

* Clippy & version

Author: dynco-nym

Cargo.lock

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "1.0.0-rc.10"
+version = "1.0.0"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -16,11 +16,13 @@ rust-version.workspace = true
 ammonia = { workspace = true }
 anyhow = { workspace = true }
 axum = { workspace = true, features = ["tokio", "macros"] }
+bip39 = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["cargo", "derive", "env", "string"] }
 cosmwasm-std = { workspace = true }
 envy = { workspace = true }
 futures-util = { workspace = true }
+itertools = { workspace = true }
 moka = { workspace = true, features = ["future"] }
 nym-contracts-common = { path = "../../common/cosmwasm-smart-contracts/contracts-common" }
 nym-bin-common = { path = "../../common/bin-common", features = ["models"] }
@@ -33,6 +35,8 @@ nym-statistics-common = { path = "../../common/statistics" }
 nym-validator-client = { path = "../../common/client-libs/validator-client" }
 nym-task = { path = "../../common/task" }
 nym-node-requests = { path = "../../nym-node/nym-node-requests", features = ["openapi"] }
+rand = { workspace = true }
+rand_chacha = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use crate::{
     http::{self, models::SummaryHistory},
-    monitor::NumericalCheckedCast,
+    utils::NumericalCheckedCast,
 };
 use anyhow::Context;
 use nym_contracts_common::Percent;
@@ -16,7 +16,7 @@ use strum_macros::{EnumString, FromRepr};
 use time::{Date, OffsetDateTime};
 use utoipa::ToSchema;
 
-pub(crate) struct GatewayRecord {
+pub(crate) struct GatewayInsertRecord {
     pub(crate) identity_key: String,
     pub(crate) bonded: bool,
     pub(crate) self_described: String,
@@ -360,14 +360,24 @@ impl TryFrom<GatewaySessionsRecord> for http::models::SessionStats {
     }
 }
 
-pub(crate) enum MixingNodeKind {
-    LegacyMixnode,
-    NymNode,
+pub(crate) enum ScrapeNodeKind {
+    LegacyMixnode { mix_id: i64 },
+    MixingNymNode { node_id: i64 },
+    EntryExitNymNode { node_id: i64, identity_key: String },
+}
+
+impl ScrapeNodeKind {
+    pub(crate) fn node_id(&self) -> &i64 {
+        match self {
+            ScrapeNodeKind::LegacyMixnode { mix_id } => mix_id,
+            ScrapeNodeKind::MixingNymNode { node_id } => node_id,
+            ScrapeNodeKind::EntryExitNymNode { node_id, .. } => node_id,
+        }
+    }
 }
 
 pub(crate) struct ScraperNodeInfo {
-    pub node_id: i64,
-    pub node_kind: MixingNodeKind,
+    pub node_kind: ScrapeNodeKind,
     pub hosts: Vec<String>,
     pub http_api_port: i64,
 }
@@ -390,6 +400,10 @@ impl ScraperNodeInfo {
 
         urls
     }
+
+    pub(crate) fn node_id(&self) -> &i64 {
+        self.node_kind.node_id()
+    }
 }
 
 #[derive(sqlx::Decode, Debug)]

nym-node-status-api/nym-node-status-api/src/db/models.rs

@@ -1,6 +1,8 @@
+use std::collections::HashSet;
+
 use crate::{
     db::{
-        models::{GatewayDto, GatewayRecord},
+        models::{GatewayDto, GatewayInsertRecord},
         DbPool,
     },
     http::models::Gateway,
@@ -30,7 +32,7 @@ pub(crate) async fn select_gateway_identity(
 
 pub(crate) async fn insert_gateways(
     pool: &DbPool,
-    gateways: Vec<GatewayRecord>,
+    gateways: Vec<GatewayInsertRecord>,
 ) -> anyhow::Result<()> {
     let mut db = pool.acquire().await?;
     for record in gateways {
@@ -98,3 +100,21 @@ pub(crate) async fn get_all_gateways(pool: &DbPool) -> anyhow::Result<Vec<Gatewa
     tracing::trace!("Fetched {} gateways from DB", items.len());
     Ok(items)
 }
+
+pub(crate) async fn get_all_gateway_id_keys(pool: &DbPool) -> anyhow::Result<HashSet<String>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query!(
+        r#"
+            SELECT gateway_identity_key
+            FROM gateways
+            WHERE bonded = true
+        "#
+    )
+    .fetch_all(&mut *conn)
+    .await?
+    .into_iter()
+    .map(|record| record.gateway_identity_key)
+    .collect::<HashSet<_>>();
+
+    Ok(items)
+}

nym-node-status-api/nym-node-status-api/src/db/queries/gateways.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use futures_util::TryStreamExt;
 use tracing::error;
 
@@ -124,3 +126,21 @@ pub(crate) async fn get_daily_stats(pool: &DbPool) -> anyhow::Result<Vec<DailySt
 
     Ok(items)
 }
+
+pub(crate) async fn get_all_mix_ids(pool: &DbPool) -> anyhow::Result<HashSet<i64>> {
+    let mut conn = pool.acquire().await?;
+    let items = sqlx::query!(
+        r#"
+            SELECT mix_id
+            FROM mixnodes
+            WHERE bonded = true
+        "#
+    )
+    .fetch_all(&mut *conn)
+    .await?
+    .into_iter()
+    .map(|record| record.mix_id)
+    .collect::<HashSet<_>>();
+
+    Ok(items)
+}

nym-node-status-api/nym-node-status-api/src/db/queries/mixnodes.rs

@@ -8,13 +8,15 @@ pub(crate) mod scraper;
 mod summary;
 pub(crate) mod testruns;
 
-pub(crate) use gateways::{get_all_gateways, insert_gateways, select_gateway_identity};
+pub(crate) use gateways::{
+    get_all_gateway_id_keys, get_all_gateways, insert_gateways, select_gateway_identity,
+};
 pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_session_records};
 pub(crate) use misc::insert_summaries;
-pub(crate) use mixnodes::{get_all_mixnodes, get_daily_stats, insert_mixnodes};
+pub(crate) use mixnodes::{get_all_mix_ids, get_all_mixnodes, get_daily_stats, insert_mixnodes};
 pub(crate) use nym_nodes::{get_nym_nodes, insert_nym_nodes};
 pub(crate) use packet_stats::{
     get_raw_node_stats, insert_daily_node_stats, insert_node_packet_stats,
 };
-pub(crate) use scraper::{get_mixing_nodes_for_scraping, insert_scraped_node_description};
+pub(crate) use scraper::{get_nodes_for_scraping, insert_scraped_node_description};
 pub(crate) use summary::{get_summary, get_summary_history};

nym-node-status-api/nym-node-status-api/src/db/queries/mod.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 
+use anyhow::Context;
 use futures_util::TryStreamExt;
 use nym_validator_client::{client::NymNodeDetails, nym_api::SkimmedNode};
 use tracing::instrument;
@@ -9,7 +10,7 @@ use crate::{
         models::{NymNodeDto, NymNodeInsertRecord},
         DbPool,
     },
-    monitor::decimal_to_i64,
+    utils::decimal_to_i64,
 };
 
 pub(crate) async fn get_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<SkimmedNode>> {
@@ -100,7 +101,8 @@ pub(crate) async fn insert_nym_nodes(
             record.last_updated_utc,
         )
         .execute(&mut *conn)
-        .await?;
+        .await
+        .with_context(|| format!("node_id={}", record.node_id))?;
     }
 
     Ok(())

nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs

@@ -1,27 +1,26 @@
 use crate::db::{
-    models::{MixingNodeKind, NodeStats, ScraperNodeInfo},
+    models::{NodeStats, ScrapeNodeKind, ScraperNodeInfo},
     DbPool,
 };
 use anyhow::Result;
 
 pub(crate) async fn insert_node_packet_stats(
     pool: &DbPool,
-    node_id: i64,
-    node_kind: &MixingNodeKind,
+    node_kind: &ScrapeNodeKind,
     stats: &NodeStats,
     timestamp_utc: i64,
 ) -> Result<()> {
     let mut conn = pool.acquire().await?;
 
     match node_kind {
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             sqlx::query!(
                 r#"
                 INSERT INTO mixnode_packet_stats_raw (
                     mix_id, timestamp_utc, packets_received, packets_sent, packets_dropped
                 ) VALUES (?, ?, ?, ?, ?)
                 "#,
-                node_id,
+                mix_id,
                 timestamp_utc,
                 stats.packets_received,
                 stats.packets_sent,
@@ -30,7 +29,8 @@ pub(crate) async fn insert_node_packet_stats(
             .execute(&mut *conn)
             .await?;
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             sqlx::query!(
                 r#"
                 INSERT INTO nym_nodes_packet_stats_raw (
@@ -60,7 +60,7 @@ pub(crate) async fn get_raw_node_stats(
     let packets = match node.node_kind {
         // if no packets are found, it's fine to assume 0 because that's also
         // SQL default value if none provided
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             sqlx::query_as!(
                 NodeStats,
                 r#"
@@ -73,12 +73,13 @@ pub(crate) async fn get_raw_node_stats(
                 ORDER BY timestamp_utc DESC
                 LIMIT 1 OFFSET 1
                 "#,
-                node.node_id
+                mix_id
             )
             .fetch_optional(&mut *conn)
             .await?
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             sqlx::query_as!(
                 NodeStats,
                 r#"
@@ -91,7 +92,7 @@ pub(crate) async fn get_raw_node_stats(
                 ORDER BY timestamp_utc DESC
                 LIMIT 1 OFFSET 1
                 "#,
-                node.node_id
+                node_id
             )
             .fetch_optional(&mut *conn)
             .await?
@@ -110,15 +111,15 @@ pub(crate) async fn insert_daily_node_stats(
     let mut conn = pool.acquire().await?;
 
     match node.node_kind {
-        MixingNodeKind::LegacyMixnode => {
+        ScrapeNodeKind::LegacyMixnode { mix_id } => {
             let total_stake = sqlx::query_scalar!(
                 r#"
                     SELECT
                         total_stake
                     FROM mixnodes
                     WHERE mix_id = ?
                    "#,
-                node.node_id
+                mix_id
             )
             .fetch_one(&mut *conn)
             .await?;
@@ -136,7 +137,7 @@ pub(crate) async fn insert_daily_node_stats(
                     packets_sent = mixnode_daily_stats.packets_sent + excluded.packets_sent,
                     packets_dropped = mixnode_daily_stats.packets_dropped + excluded.packets_dropped
                 "#,
-                node.node_id,
+                mix_id,
                 date_utc,
                 total_stake,
                 packets.packets_received,
@@ -146,15 +147,16 @@ pub(crate) async fn insert_daily_node_stats(
             .execute(&mut *conn)
             .await?;
         }
-        MixingNodeKind::NymNode => {
+        ScrapeNodeKind::MixingNymNode { node_id }
+        | ScrapeNodeKind::EntryExitNymNode { node_id, .. } => {
             let total_stake = sqlx::query_scalar!(
                 r#"
                 SELECT
                     total_stake
                 FROM nym_nodes
                 WHERE node_id = ?
                 "#,
-                node.node_id
+                node_id
             )
             .fetch_one(&mut *conn)
             .await?;
@@ -172,7 +174,7 @@ pub(crate) async fn insert_daily_node_stats(
                     packets_sent = nym_node_daily_mixing_stats.packets_sent + excluded.packets_sent,
                     packets_dropped = nym_node_daily_mixing_stats.packets_dropped + excluded.packets_dropped
                 "#,
-                node.node_id,
+                node_id,
                 date_utc,
                 total_stake,
                 packets.packets_received,