[feat] Socks5 and Native client: run with hardcoded topology (#3866)

* allow running clients using hardcoded topology

* fixed sdk/lib/socks5-listener build

* fixed nym-connect build

* allow for both snake_case and camelCase deserialization

Author: jstuczyn

Cargo.lock

@@ -21,6 +21,7 @@ use nym_task::connections::TransmissionLane;
 use nym_task::TaskManager;
 use nym_validator_client::QueryHttpRpcNyxdClient;
 use std::error::Error;
+use std::path::PathBuf;
 use tokio::sync::watch::error::SendError;
 
 pub use nym_sphinx::addressing::clients::Recipient;
@@ -34,11 +35,17 @@ pub struct SocketClient {
     /// Client configuration options, including, among other things, packet sending rates,
     /// key filepaths, etc.
     config: Config,
+
+    /// Optional path to a .json file containing standalone network details.
+    custom_mixnet: Option<PathBuf>,
 }
 
 impl SocketClient {
-    pub fn new(config: Config) -> Self {
-        SocketClient { config }
+    pub fn new(config: Config, custom_mixnet: Option<PathBuf>) -> Self {
+        SocketClient {
+            config,
+            custom_mixnet,
+        }
     }
 
     fn start_websocket_listener(
@@ -109,7 +116,11 @@ impl SocketClient {
 
         let storage = self.initialise_storage().await?;
 
-        let base_client = BaseClientBuilder::new(&self.config.base, storage, dkg_query_client);
+        let mut base_client = BaseClientBuilder::new(&self.config.base, storage, dkg_query_client);
+
+        if let Some(custom_mixnet) = &self.custom_mixnet {
+            base_client = base_client.with_stored_topology(custom_mixnet)?;
+        }
 
         Ok(base_client)
     }

clients/native/src/client/mod.rs

@@ -15,12 +15,15 @@ use nym_bin_common::output_format::OutputFormat;
 use nym_client_core::client::base_client::storage::gateway_details::OnDiskGatewayDetails;
 use nym_client_core::client::key_manager::persistence::OnDiskKeys;
 use nym_client_core::config::GatewayEndpointConfig;
+use nym_client_core::init::helpers::current_gateways;
 use nym_client_core::init::GatewaySetup;
 use nym_crypto::asymmetric::identity;
 use nym_sphinx::addressing::clients::Recipient;
+use nym_topology::NymTopology;
 use serde::Serialize;
 use std::fmt::Display;
 use std::net::IpAddr;
+use std::path::PathBuf;
 use std::{fs, io};
 use tap::TapFallible;
 
@@ -49,7 +52,12 @@ pub(crate) struct Init {
     nyxd_urls: Option<Vec<url::Url>>,
 
     /// Comma separated list of rest endpoints of the API validators
-    #[clap(long, alias = "api_validators", value_delimiter = ',')]
+    #[clap(
+        long,
+        alias = "api_validators",
+        value_delimiter = ',',
+        group = "network"
+    )]
     // the alias here is included for backwards compatibility (1.1.4 and before)
     nym_apis: Option<Vec<url::Url>>,
 
@@ -65,6 +73,10 @@ pub(crate) struct Init {
     #[clap(long)]
     host: Option<IpAddr>,
 
+    /// Path to .json file containing custom network specification.
+    #[clap(long, group = "network", hide = true)]
+    custom_mixnet: Option<PathBuf>,
+
     /// Mostly debug-related option to increase default traffic rate so that you would not need to
     /// modify config post init
     #[clap(long, hide = true)]
@@ -130,7 +142,7 @@ fn init_paths(id: &str) -> io::Result<()> {
     fs::create_dir_all(default_config_directory(id))
 }
 
-pub(crate) async fn execute(args: &Init) -> Result<(), ClientError> {
+pub(crate) async fn execute(args: Init) -> Result<(), ClientError> {
     eprintln!("Initialising client...");
 
     let id = &args.id;
@@ -173,12 +185,25 @@ pub(crate) async fn execute(args: &Init) -> Result<(), ClientError> {
     let key_store = OnDiskKeys::new(config.storage_paths.common_paths.keys.clone());
     let details_store =
         OnDiskGatewayDetails::new(&config.storage_paths.common_paths.gateway_details);
-    let init_details = nym_client_core::init::setup_gateway(
+
+    let network_gateways = if let Some(hardcoded_topology) = args
+        .custom_mixnet
+        .map(NymTopology::new_from_file)
+        .transpose()?
+    {
+        // hardcoded_topology
+        hardcoded_topology.get_gateways()
+    } else {
+        let mut rng = rand::thread_rng();
+        current_gateways(&mut rng, &config.base.client.nym_api_urls).await?
+    };
+
+    let init_details = nym_client_core::init::setup_gateway_from(
         gateway_setup,
         &key_store,
         &details_store,
         register_gateway,
-        Some(&config.base.client.nym_api_urls),
+        Some(&network_gateways),
     )
     .await
     .tap_err(|err| eprintln!("Failed to setup gateway\nError: {err}"))?

clients/native/src/commands/init.rs

@@ -84,8 +84,8 @@ pub(crate) async fn execute(args: Cli) -> Result<(), Box<dyn Error + Send + Sync
     let bin_name = "nym-native-client";
 
     match args.command {
-        Commands::Init(m) => init::execute(&m).await?,
-        Commands::Run(m) => run::execute(&m).await?,
+        Commands::Init(m) => init::execute(m).await?,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::BuildInfo(m) => build_info::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::command(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::command(), bin_name),

clients/native/src/commands/mod.rs

@@ -13,6 +13,7 @@ use nym_bin_common::version_checker::is_minor_version_compatible;
 use nym_crypto::asymmetric::identity;
 use std::error::Error;
 use std::net::IpAddr;
+use std::path::PathBuf;
 
 #[derive(Args, Clone)]
 pub(crate) struct Run {
@@ -25,7 +26,12 @@ pub(crate) struct Run {
     nyxd_urls: Option<Vec<url::Url>>,
 
     /// Comma separated list of rest endpoints of the API validators
-    #[clap(long, alias = "api_validators", value_delimiter = ',')]
+    #[clap(
+        long,
+        alias = "api_validators",
+        value_delimiter = ',',
+        group = "network"
+    )]
     // the alias here is included for backwards compatibility (1.1.4 and before)
     nym_apis: Option<Vec<url::Url>>,
 
@@ -46,6 +52,10 @@ pub(crate) struct Run {
     #[clap(long)]
     host: Option<IpAddr>,
 
+    /// Path to .json file containing custom network specification.
+    #[clap(long, group = "network", hide = true)]
+    custom_mixnet: Option<PathBuf>,
+
     /// Mostly debug-related option to increase default traffic rate so that you would not need to
     /// modify config post init
     #[clap(long, hide = true)]
@@ -95,7 +105,7 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) -> Result<(), Box<dyn Error + Send + Sync>> {
+pub(crate) async fn execute(args: Run) -> Result<(), Box<dyn Error + Send + Sync>> {
     eprintln!("Starting client {}...", args.id);
 
     let mut config = try_load_current_config(&args.id)?;
@@ -106,5 +116,7 @@ pub(crate) async fn execute(args: &Run) -> Result<(), Box<dyn Error + Send + Syn
         return Err(Box::new(ClientError::FailedLocalVersionCheck));
     }
 
-    SocketClient::new(config).run_socket_forever().await
+    SocketClient::new(config, args.custom_mixnet)
+        .run_socket_forever()
+        .await
 }

clients/native/src/commands/run.rs

@@ -230,8 +230,7 @@ impl ServerResponse {
 
         let error_kind = ErrorKind::try_from(b[1])?;
 
-        let message_len =
-            u64::from_be_bytes(b[2..2 + size_of::<u64>()].as_ref().try_into().unwrap());
+        let message_len = u64::from_be_bytes(b[2..2 + size_of::<u64>()].try_into().unwrap());
         let message = &b[2 + size_of::<u64>()..];
         if message.len() as u64 != message_len {
             return Err(error::Error::new(

clients/native/websocket-requests/src/responses.rs

@@ -16,6 +16,7 @@ serde_json = { workspace = true }
 tap = "1.0.1"
 thiserror = { workspace = true }
 tokio = { version = "1.24.1", features = ["rt-multi-thread", "net", "signal"] }
+rand = "0.7.3"
 url = { workspace = true }
 
 # internal

clients/socks5/Cargo.toml

@@ -14,11 +14,14 @@ use nym_bin_common::output_format::OutputFormat;
 use nym_client_core::client::base_client::storage::gateway_details::OnDiskGatewayDetails;
 use nym_client_core::client::key_manager::persistence::OnDiskKeys;
 use nym_client_core::config::GatewayEndpointConfig;
+use nym_client_core::init::helpers::current_gateways;
 use nym_client_core::init::GatewaySetup;
 use nym_crypto::asymmetric::identity;
 use nym_sphinx::addressing::clients::Recipient;
+use nym_topology::NymTopology;
 use serde::Serialize;
 use std::fmt::Display;
+use std::path::PathBuf;
 use std::{fs, io};
 use tap::TapFallible;
 
@@ -68,6 +71,10 @@ pub(crate) struct Init {
     #[clap(short, long)]
     port: Option<u16>,
 
+    /// Path to .json file containing custom network specification.
+    #[clap(long, group = "network", hide = true)]
+    custom_mixnet: Option<PathBuf>,
+
     /// Mostly debug-related option to increase default traffic rate so that you would not need to
     /// modify config post init
     #[clap(long, hide = true)]
@@ -138,7 +145,7 @@ fn init_paths(id: &str) -> io::Result<()> {
     fs::create_dir_all(default_config_directory(id))
 }
 
-pub(crate) async fn execute(args: &Init) -> Result<(), Socks5ClientError> {
+pub(crate) async fn execute(args: Init) -> Result<(), Socks5ClientError> {
     eprintln!("Initialising client...");
 
     let id = &args.id;
@@ -185,12 +192,25 @@ pub(crate) async fn execute(args: &Init) -> Result<(), Socks5ClientError> {
     let key_store = OnDiskKeys::new(config.storage_paths.common_paths.keys.clone());
     let details_store =
         OnDiskGatewayDetails::new(&config.storage_paths.common_paths.gateway_details);
-    let init_details = nym_client_core::init::setup_gateway(
+
+    let network_gateways = if let Some(hardcoded_topology) = args
+        .custom_mixnet
+        .map(NymTopology::new_from_file)
+        .transpose()?
+    {
+        // hardcoded_topology
+        hardcoded_topology.get_gateways()
+    } else {
+        let mut rng = rand::thread_rng();
+        current_gateways(&mut rng, &config.core.base.client.nym_api_urls).await?
+    };
+
+    let init_details = nym_client_core::init::setup_gateway_from(
         gateway_setup,
         &key_store,
         &details_store,
         register_gateway,
-        Some(&config.core.base.client.nym_api_urls),
+        Some(&network_gateways),
     )
     .await
     .tap_err(|err| eprintln!("Failed to setup gateway\nError: {err}"))?

clients/socks5/src/commands/init.rs

@@ -87,8 +87,8 @@ pub(crate) async fn execute(args: Cli) -> Result<(), Box<dyn Error + Send + Sync
     let bin_name = "nym-socks5-client";
 
     match args.command {
-        Commands::Init(m) => init::execute(&m).await?,
-        Commands::Run(m) => run::execute(&m).await?,
+        Commands::Init(m) => init::execute(m).await?,
+        Commands::Run(m) => run::execute(m).await?,
         Commands::BuildInfo(m) => build_info::execute(m),
         Commands::Completions(s) => s.generate(&mut Cli::command(), bin_name),
         Commands::GenerateFigSpec => fig_generate(&mut Cli::command(), bin_name),

clients/socks5/src/commands/mod.rs

@@ -15,6 +15,7 @@ use nym_client_core::client::topology_control::geo_aware_provider::CountryGroup;
 use nym_crypto::asymmetric::identity;
 use nym_socks5_client_core::NymClient;
 use nym_sphinx::addressing::clients::Recipient;
+use std::path::PathBuf;
 
 #[derive(Args, Clone)]
 pub(crate) struct Run {
@@ -45,13 +46,17 @@ pub(crate) struct Run {
     nyxd_urls: Option<Vec<url::Url>>,
 
     /// Comma separated list of rest endpoints of the Nym APIs
-    #[clap(long, value_delimiter = ',')]
+    #[clap(long, value_delimiter = ',', group = "network")]
     nym_apis: Option<Vec<url::Url>>,
 
     /// Port for the socket to listen on
     #[clap(short, long)]
     port: Option<u16>,
 
+    /// Path to .json file containing custom network specification.
+    #[clap(long, group = "network", group = "routing", hide = true)]
+    custom_mixnet: Option<PathBuf>,
+
     /// Mostly debug-related option to increase default traffic rate so that you would not need to
     /// modify config post init
     #[clap(long, hide = true)]
@@ -62,7 +67,7 @@ pub(crate) struct Run {
     no_cover: bool,
 
     /// Set geo-aware mixnode selection when sending mixnet traffic, for experiments only.
-    #[clap(long, hide = true, value_parser = validate_country_group)]
+    #[clap(long, hide = true, value_parser = validate_country_group, group="routing")]
     geo_routing: Option<CountryGroup>,
 
     /// Enable medium mixnet traffic, for experiments only.
@@ -124,7 +129,7 @@ fn version_check(cfg: &Config) -> bool {
     }
 }
 
-pub(crate) async fn execute(args: &Run) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+pub(crate) async fn execute(args: Run) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     eprintln!("Starting client {}...", args.id);
 
     let mut config = try_load_current_config(&args.id)?;
@@ -138,5 +143,7 @@ pub(crate) async fn execute(args: &Run) -> Result<(), Box<dyn std::error::Error
     let storage =
         OnDiskPersistent::from_paths(config.storage_paths.common_paths, &config.core.base.debug)
             .await?;
-    NymClient::new(config.core, storage).run_forever().await
+    NymClient::new(config.core, storage, args.custom_mixnet)
+        .run_forever()
+        .await
 }

clients/socks5/src/commands/run.rs

@@ -40,7 +40,7 @@ nym-gateway-requests = { path = "../../gateway/gateway-requests" }
 nym-nonexhaustive-delayqueue = { path = "../nonexhaustive-delayqueue" }
 nym-sphinx = { path = "../nymsphinx" }
 nym-pemstore = { path = "../pemstore" }
-nym-topology = { path = "../topology" }
+nym-topology = { path = "../topology", features = ["serializable"] }
 nym-validator-client = { path = "../client-libs/validator-client", default-features = false }
 nym-task = { path = "../task" }
 nym-credential-storage = { path = "../credential-storage" }