Auto merge of #508 - Zeegomo:last-crates-update, r=pietroalbini

Last crates update

Add a cron job to update the crates lists every 24h and a related metric to get the time of last update.

Author: bors

src/server/cronjobs.rs

@@ -0,0 +1,53 @@
+use crate::actions::{Action, ActionsCtx, UpdateLists};
+use crate::prelude::*;
+use crate::server::Data;
+use crate::utils;
+use std::sync::Arc;
+use std::thread;
+use std::time::Duration;
+
+const DAY: Duration = Duration::from_secs(60 * 60 * 24);
+struct JobDescription {
+    name: &'static str,
+    interval: Duration,
+    exec: fn(Arc<Data>) -> Fallible<()>,
+}
+
+static JOBS: &[JobDescription] = &[JobDescription {
+    name: "crates lists update",
+    interval: DAY,
+    exec: update_crates as fn(Arc<Data>) -> Fallible<()>,
+}];
+
+pub fn spawn(data: Data) {
+    let data = Arc::new(data);
+    for job in JOBS {
+        // needed to make the borrowck happy
+        let data = Arc::clone(&data);
+
+        thread::spawn(move || loop {
+            let result = (job.exec)(Arc::clone(&data));
+            if let Err(e) = result {
+                utils::report_failure(&e);
+            }
+
+            info!(
+                "the {} thread will be respawned in {}s",
+                job.name,
+                job.interval.as_secs()
+            );
+            thread::sleep(job.interval);
+        });
+    }
+}
+
+fn update_crates(data: Arc<Data>) -> Fallible<()> {
+    let ctx = ActionsCtx::new(&data.db, &data.config);
+
+    UpdateLists {
+        github: true,
+        registry: true,
+        local: false,
+    }
+    .apply(&ctx)
+}

src/server/metrics.rs

@@ -1,16 +1,23 @@
-use crate::db::Database;
+use crate::db::{Database, QueryUtils};
 use crate::experiments::{Assignee, Experiment};
 use crate::prelude::*;
 use crate::server::agents::Agent;
+use chrono::{DateTime, Utc};
 use prometheus::proto::{Metric, MetricFamily};
-use prometheus::{IntCounterVec, IntGaugeVec, __register_counter_vec, __register_gauge_vec};
+use prometheus::{
+    IntCounterVec, IntGauge, IntGaugeVec, __register_counter_vec, __register_gauge,
+    __register_gauge_vec,
+};
+
 const JOBS_METRIC: &str = "crater_completed_jobs_total";
 const AGENT_WORK_METRIC: &str = "crater_agent_supposed_to_work";
+const LAST_CRATES_UPDATE_METRIC: &str = "crater_last_crates_update";
 
 #[derive(Clone)]
 pub struct Metrics {
     crater_completed_jobs_total: IntCounterVec,
     crater_work_status: IntGaugeVec,
+    crater_last_crates_update: IntGauge,
 }
 
 impl Metrics {
@@ -20,10 +27,14 @@ impl Metrics {
             prometheus::register_int_counter_vec!(jobs_opts, &["agent", "experiment"])?;
         let agent_opts = prometheus::opts!(AGENT_WORK_METRIC, "is agent supposed to work");
         let crater_work_status = prometheus::register_int_gauge_vec!(agent_opts, &["agent"])?;
+        let crates_update_opts =
+            prometheus::opts!(LAST_CRATES_UPDATE_METRIC, "last update of crates lists");
+        let crater_last_crates_update = prometheus::register_int_gauge!(crates_update_opts)?;
 
         Ok(Metrics {
             crater_completed_jobs_total,
             crater_work_status,
+            crater_last_crates_update,
         })
     }
 
@@ -79,20 +90,36 @@ impl Metrics {
         Ok(())
     }
 
+    pub fn update_crates_lists(&self, db: &Database) -> Fallible<()> {
+        //&[] as &[u32] is just a hint to make the compiler happy
+        let datetime: Option<DateTime<Utc>> =
+            db.get_row("SELECT MAX(loaded_at) FROM crates;", &[] as &[u32], |r| {
+                r.get(0)
+            })?;
+
+        if let Some(datetime) = datetime {
+            self.crater_last_crates_update.set(datetime.timestamp());
+            Ok(())
+        } else {
+            bail!("no crates loaded");
+        }
+    }
+
     pub fn on_complete_experiment(&self, experiment: &str) -> Fallible<()> {
         self.remove_experiment_jobs(experiment)
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{Metrics, AGENT_WORK_METRIC, JOBS_METRIC};
+    use super::{Metrics, AGENT_WORK_METRIC, JOBS_METRIC, LAST_CRATES_UPDATE_METRIC};
     use crate::actions::{Action, ActionsCtx, CreateExperiment, EditExperiment};
     use crate::config::Config;
     use crate::db::Database;
     use crate::experiments::{Assignee, Experiment};
     use crate::server::agents::{Agent, Agents};
     use crate::server::tokens::Tokens;
+    use chrono::Utc;
     use lazy_static::lazy_static;
     use prometheus::proto::MetricFamily;
 
@@ -202,4 +229,20 @@ mod tests {
         assert!(supposed_to_work(&status, Some(agent1)));
         assert!(!supposed_to_work(&status, Some(agent2)));
     }
+
+    #[test]
+    fn test_crates_list_update() {
+        let db = Database::temp().unwrap();
+        let config = Config::default();
+
+        let now = Utc::now();
+        crate::crates::lists::setup_test_lists(&db, &config).unwrap();
+        METRICS.update_crates_lists(&db).unwrap();
+        let last_update = Metrics::get_metric_by_name(LAST_CRATES_UPDATE_METRIC)
+            .unwrap()
+            .get_metric()[0]
+            .get_gauge()
+            .get_value() as i64;
+        assert!(last_update >= now.timestamp());
+    }
 }

src/server/mod.rs

@@ -1,6 +1,7 @@
 pub mod agents;
 pub mod api_types;
 mod auth;
+mod cronjobs;
 mod github;
 mod messages;
 mod metrics;
@@ -74,6 +75,7 @@ pub fn run(config: Config) -> Fallible<()> {
     let mutex = Arc::new(Mutex::new(data.clone()));
 
     data.reports_worker.spawn(data.clone());
+    cronjobs::spawn(data.clone());
 
     info!("running server...");
 

src/server/routes/metrics.rs

@@ -33,6 +33,9 @@ fn endpoint_metrics(data: Arc<Data>) -> Fallible<Response<Body>> {
         &data.db,
         &data.agents.all()?.iter().collect::<Vec<&Agent>>(),
     )?;
+
+    data.metrics.update_crates_lists(&data.db)?;
+
     let mut buffer = Vec::new();
     let families = prometheus::gather();
     TextEncoder::new().encode(&families, &mut buffer)?;