worker: add a job to invalidate paths on our CDNs

Author: LawnGnome

src/worker/jobs/invalidate_cdns.rs

@@ -0,0 +1,57 @@
+use std::sync::Arc;
+
+use anyhow::Context;
+use crates_io_worker::BackgroundJob;
+
+use crate::worker::Environment;
+
+/// A background job that invalidates the given paths on all CDNs in use on crates.io.
+#[derive(Deserialize, Serialize)]
+pub struct InvalidateCdns {
+    paths: Vec<String>,
+}
+
+impl InvalidateCdns {
+    pub fn new<I>(paths: I) -> Self
+    where
+        I: Iterator,
+        I::Item: ToString,
+    {
+        Self {
+            paths: paths.map(|path| path.to_string()).collect(),
+        }
+    }
+}
+
+impl BackgroundJob for InvalidateCdns {
+    const JOB_NAME: &'static str = "invalidate_cdns";
+
+    type Context = Arc<Environment>;
+
+    async fn run(&self, ctx: Self::Context) -> anyhow::Result<()> {
+        // Fastly doesn't provide an API to purge multiple paths at once, except through the use of
+        // surrogate keys. We can't use surrogate keys right now because they require a
+        // Fastly-specific header, and not all of our traffic goes through Fastly.
+        //
+        // For now, we won't parallelise: most crate deletions are for new crates with one (or very
+        // few) versions, so the actual number of paths being invalidated is likely to be small, and
+        // this is all happening from either a background job or admin command anyway.
+        if let Some(fastly) = ctx.fastly() {
+            for path in self.paths.iter() {
+                fastly
+                    .invalidate(path)
+                    .await
+                    .with_context(|| format!("Failed to invalidate path on Fastly CDN: {path}"))?;
+            }
+        }
+
+        if let Some(cloudfront) = ctx.cloudfront() {
+            cloudfront
+                .invalidate_many(self.paths.clone())
+                .await
+                .context("Failed to invalidate paths on CloudFront CDN")?;
+        }
+
+        Ok(())
+    }
+}

src/worker/jobs/mod.rs

@@ -6,6 +6,7 @@ pub mod dump_db;
 mod expiry_notification;
 mod index;
 mod index_version_downloads_archive;
+mod invalidate_cdns;
 mod readmes;
 pub mod rss;
 mod send_publish_notifications;
@@ -23,6 +24,7 @@ pub use self::dump_db::DumpDb;
 pub use self::expiry_notification::SendTokenExpiryNotifications;
 pub use self::index::{NormalizeIndex, SquashIndex, SyncToGitIndex, SyncToSparseIndex};
 pub use self::index_version_downloads_archive::IndexVersionDownloadsArchive;
+pub use self::invalidate_cdns::InvalidateCdns;
 pub use self::readmes::RenderAndUploadReadme;
 pub use self::send_publish_notifications::SendPublishNotificationsJob;
 pub use self::sync_admins::SyncAdmins;

src/worker/mod.rs

@@ -26,6 +26,7 @@ impl RunnerExt for Runner<Arc<Environment>> {
             .register_job_type::<jobs::DeleteCrateFromStorage>()
             .register_job_type::<jobs::DumpDb>()
             .register_job_type::<jobs::IndexVersionDownloadsArchive>()
+            .register_job_type::<jobs::InvalidateCdns>()
             .register_job_type::<jobs::NormalizeIndex>()
             .register_job_type::<jobs::ProcessCdnLog>()
             .register_job_type::<jobs::ProcessCdnLogQueue>()