Explorer API: add new HTTP resource to decorate mix nodes with geoip locations (#734)

mmsinclair · web-flow · commit ccf8063a5206 · 2021-08-11T17:24:34.000+01:00
* explorer-api: decorate mix nodes with locations from the geoip service and keep mix node cache in a hash map instead of a vec

* explorer-api: add `lat` and `lng` for map views

* explorer-api: remove function and simplify code

* explorer-api: review feedback

* network-explorer: format
diff --git a/explorer-api/src/country_statistics/country_nodes_distribution.rs b/explorer-api/src/country_statistics/country_nodes_distribution.rs
@@ -16,7 +16,9 @@ impl ConcurrentCountryNodesDistribution {
         }
     }
 
-    pub(crate) fn attach(country_node_distribution: CountryNodesDistribution) -> Self {
+    pub(crate) fn new_from_distribution(
+        country_node_distribution: CountryNodesDistribution,
+    ) -> Self {
         ConcurrentCountryNodesDistribution {
             inner: Arc::new(RwLock::new(country_node_distribution)),
         }
diff --git a/explorer-api/src/country_statistics/mod.rs b/explorer-api/src/country_statistics/mod.rs
@@ -1,15 +1,12 @@
-use isocountry::CountryCode;
 use log::{info, trace, warn};
 use reqwest::Error as ReqwestError;
 
-use models::GeoLocation;
-
 use crate::country_statistics::country_nodes_distribution::CountryNodesDistribution;
+use crate::mix_nodes::{GeoLocation, Location};
 use crate::state::ExplorerApiStateContext;
 
 pub mod country_nodes_distribution;
 pub mod http;
-mod models;
 
 pub(crate) struct CountryStatistics {
     state: ExplorerApiStateContext,
@@ -38,30 +35,31 @@ impl CountryStatistics {
     /// Retrieves the current list of mixnodes from the validators and calculates how many nodes are in each country
     async fn calculate_nodes_per_country(&mut self) {
         // force the mixnode cache to invalidate
-        let mixnode_bonds = self
-            .state
-            .inner
-            .mix_nodes
-            .clone()
-            .refresh_and_get()
-            .await
-            .value;
+        let mixnode_bonds = self.state.inner.mix_nodes.refresh_and_get().await.value;
 
         let mut distribution = CountryNodesDistribution::new();
 
         info!("Locating mixnodes...");
-        for (i, bond) in mixnode_bonds.iter().enumerate() {
-            match locate(&bond.mix_node.host).await {
-                Ok(location) => {
-                    let country_code = map_2_letter_to_3_letter_country_code(&location);
-                    *(distribution.entry(country_code)).or_insert(0) += 1;
+        for (i, cache_item) in mixnode_bonds.values().enumerate() {
+            match locate(&cache_item.bond.mix_node.host).await {
+                Ok(geo_location) => {
+                    let location = Location::new(geo_location);
+
+                    *(distribution.entry(location.three_letter_iso_country_code.to_string()))
+                        .or_insert(0) += 1;
 
                     trace!(
                         "Ip {} is located in {:#?}",
-                        bond.mix_node.host,
-                        map_2_letter_to_3_letter_country_code(&location)
+                        cache_item.bond.mix_node.host,
+                        location.three_letter_iso_country_code,
                     );
 
+                    self.state
+                        .inner
+                        .mix_nodes
+                        .set_location(&cache_item.bond.mix_node.identity_key, location)
+                        .await;
+
                     if (i % 100) == 0 {
                         info!(
                             "Located {} mixnodes in {} countries",
@@ -91,19 +89,6 @@ impl CountryStatistics {
     }
 }
 
-fn map_2_letter_to_3_letter_country_code(geo: &GeoLocation) -> String {
-    match CountryCode::for_alpha2(&geo.country_code) {
-        Ok(three_letter_country_code) => three_letter_country_code.alpha3().to_string(),
-        Err(_e) => {
-            warn!(
-                "❌ Oh no! map_2_letter_to_3_letter_country_code failed for '{:#?}'",
-                geo
-            );
-            "???".to_string()
-        }
-    }
-}
-
 async fn locate(ip: &str) -> Result<GeoLocation, ReqwestError> {
     let response = reqwest::get(format!("{}{}", crate::GEO_IP_SERVICE, ip)).await?;
     let location = response.json::<GeoLocation>().await?;
diff --git a/explorer-api/src/country_statistics/models.rs b/explorer-api/src/country_statistics/models.rs
diff --git a/explorer-api/src/mix_node/http.rs b/explorer-api/src/mix_node/http.rs
@@ -1,13 +1,54 @@
 use reqwest::Error as ReqwestError;
-
 use rocket::serde::json::Json;
 use rocket::{Route, State};
+use serde::Serialize;
+
+use mixnet_contract::{Addr, Coin, Layer, MixNode};
 
 use crate::mix_node::models::{NodeDescription, NodeStats};
+use crate::mix_nodes::Location;
 use crate::state::ExplorerApiStateContext;
 
 pub fn mix_node_make_default_routes() -> Vec<Route> {
-    routes_with_openapi![get_description, get_stats]
+    routes_with_openapi![get_description, get_stats, list]
+}
+
+#[derive(Clone, Debug, Serialize, JsonSchema)]
+pub(crate) struct PrettyMixNodeBondWithLocation {
+    pub location: Option<Location>,
+    pub bond_amount: Coin,
+    pub total_delegation: Coin,
+    pub owner: Addr,
+    pub layer: Layer,
+    pub mix_node: MixNode,
+}
+
+#[openapi(tag = "mix_node")]
+#[get("/")]
+pub(crate) async fn list(
+    state: &State<ExplorerApiStateContext>,
+) -> Json<Vec<PrettyMixNodeBondWithLocation>> {
+    Json(
+        state
+            .inner
+            .mix_nodes
+            .get()
+            .await
+            .value
+            .values()
+            .map(|i| {
+                let mix_node = i.bond.clone();
+                PrettyMixNodeBondWithLocation {
+                    location: i.location.clone(),
+                    bond_amount: mix_node.bond_amount,
+                    total_delegation: mix_node.total_delegation,
+                    owner: mix_node.owner,
+                    layer: mix_node.layer,
+                    mix_node: mix_node.mix_node,
+                }
+            })
+            .collect::<Vec<PrettyMixNodeBondWithLocation>>(),
+    )
 }
 
 #[openapi(tag = "mix_node")]
diff --git a/explorer-api/src/mix_nodes/mod.rs b/explorer-api/src/mix_nodes/mod.rs
@@ -1,15 +1,66 @@
+mod utils;
+
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use rocket::tokio::sync::RwLock;
+use serde::{Deserialize, Serialize};
 
+use crate::mix_nodes::utils::map_2_letter_to_3_letter_country_code;
 use mixnet_contract::MixNodeBond;
 use validator_client::Config;
 
+pub(crate) type LocationCache = HashMap<String, Location>;
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct GeoLocation {
+    pub(crate) ip: String,
+    pub(crate) country_code: String,
+    pub(crate) country_name: String,
+    pub(crate) region_code: String,
+    pub(crate) region_name: String,
+    pub(crate) city: String,
+    pub(crate) zip_code: String,
+    pub(crate) time_zone: String,
+    pub(crate) latitude: f32,
+    pub(crate) longitude: f32,
+    pub(crate) metro_code: u32,
+}
+
+#[derive(Clone, Debug, JsonSchema, Serialize, Deserialize)]
+pub(crate) struct Location {
+    pub(crate) two_letter_iso_country_code: String,
+    pub(crate) three_letter_iso_country_code: String,
+    pub(crate) country_name: String,
+    pub(crate) lat: f32,
+    pub(crate) lng: f32,
+}
+
+impl Location {
+    pub(crate) fn new(geo_location: GeoLocation) -> Self {
+        let three_letter_iso_country_code = map_2_letter_to_3_letter_country_code(&geo_location);
+        Location {
+            country_name: geo_location.country_name,
+            two_letter_iso_country_code: geo_location.country_code,
+            three_letter_iso_country_code,
+            lat: geo_location.latitude,
+            lng: geo_location.longitude,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MixNodeBondWithLocation {
+    pub(crate) location: Option<Location>,
+    pub(crate) bond: MixNodeBond,
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct MixNodesResult {
     pub(crate) valid_until: SystemTime,
-    pub(crate) value: Vec<MixNodeBond>,
+    pub(crate) value: HashMap<String, MixNodeBondWithLocation>,
+    location_cache: LocationCache,
 }
 
 #[derive(Clone)]
@@ -21,23 +72,53 @@ impl ThreadsafeMixNodesResult {
     pub(crate) fn new() -> Self {
         ThreadsafeMixNodesResult {
             inner: Arc::new(RwLock::new(MixNodesResult {
-                value: vec![],
+                value: HashMap::new(),
                 valid_until: SystemTime::now() - Duration::from_secs(60), // in the past
+                location_cache: LocationCache::new(),
             })),
         }
     }
 
+    pub(crate) fn new_with_location_cache(location_cache: LocationCache) -> Self {
+        ThreadsafeMixNodesResult {
+            inner: Arc::new(RwLock::new(MixNodesResult {
+                value: HashMap::new(),
+                valid_until: SystemTime::now() - Duration::from_secs(60), // in the past
+                location_cache,
+            })),
+        }
+    }
+
+    pub(crate) async fn get_location_cache(&self) -> LocationCache {
+        self.inner.read().await.location_cache.clone()
+    }
+
+    pub(crate) async fn set_location(&self, identity_key: &str, location: Location) {
+        let mut guard = self.inner.write().await;
+
+        // cache the location for this mix node so that it can be used when the mix node list is refreshed
+        guard
+            .location_cache
+            .insert(identity_key.to_string(), location.clone());
+
+        // add the location to the mix node
+        guard
+            .value
+            .entry(identity_key.to_string())
+            .and_modify(|item| item.location = Some(location));
+    }
+
     pub(crate) async fn get(&self) -> MixNodesResult {
         // check ttl
-        let valid_until = self.inner.clone().read().await.valid_until;
+        let valid_until = self.inner.read().await.valid_until;
 
         if valid_until < SystemTime::now() {
             // force reload
             self.refresh().await;
         }
 
         // return in-memory cache
-        self.inner.clone().read().await.clone()
+        self.inner.read().await.clone()
     }
 
     pub(crate) async fn refresh_and_get(&self) -> MixNodesResult {
@@ -48,10 +129,21 @@ impl ThreadsafeMixNodesResult {
     async fn refresh(&self) {
         // get mixnodes and cache the new value
         let value = retrieve_mixnodes().await;
-        self.inner.write().await.clone_from(&MixNodesResult {
-            value,
+        let location_cache = self.inner.read().await.location_cache.clone();
+        *self.inner.write().await = MixNodesResult {
+            value: value
+                .into_iter()
+                .map(|bond| {
+                    let location = location_cache.get(&bond.mix_node.identity_key).cloned(); // add the location, if we've located this mix node before
+                    (
+                        bond.mix_node.identity_key.to_string(),
+                        MixNodeBondWithLocation { bond, location },
+                    )
+                })
+                .collect(),
             valid_until: SystemTime::now() + Duration::from_secs(60 * 10), // valid for 10 minutes
-        });
+            location_cache,
+        };
     }
 }
 
diff --git a/explorer-api/src/mix_nodes/utils.rs b/explorer-api/src/mix_nodes/utils.rs
@@ -0,0 +1,15 @@
+use crate::mix_nodes::GeoLocation;
+use isocountry::CountryCode;
+
+pub(crate) fn map_2_letter_to_3_letter_country_code(geo: &GeoLocation) -> String {
+    match CountryCode::for_alpha2(&geo.country_code) {
+        Ok(three_letter_country_code) => three_letter_country_code.alpha3().to_string(),
+        Err(_e) => {
+            warn!(
+                "❌ Oh no! map_2_letter_to_3_letter_country_code failed for '{:#?}'",
+                geo
+            );
+            "???".to_string()
+        }
+    }
+}
diff --git a/explorer-api/src/state.rs b/explorer-api/src/state.rs

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,9 @@ impl ConcurrentCountryNodesDistribution {`
`16`	`16`	`}`
`17`	`17`	`}`
`18`	`18`
`19`		`- pub(crate) fn attach(country_node_distribution: CountryNodesDistribution) -> Self {`
	`19`	`+ pub(crate) fn new_from_distribution(`
	`20`	`+ country_node_distribution: CountryNodesDistribution,`
	`21`	`+ ) -> Self {`
`20`	`22`	`ConcurrentCountryNodesDistribution {`
`21`	`23`	`inner: Arc::new(RwLock::new(country_node_distribution)),`
`22`	`24`	`}`