Update examples to include rules for deleting failed pending imports

primary-site/aws/modules/iam/inbox-listener.tf

@@ -32,7 +32,7 @@ data "aws_iam_policy_document" "inbox_listener_policy_document" {
 }
 
 resource "aws_iam_policy" "inbox_listener_policy" {
-  name = "${var.eks_foxglove_namespace}-inbox-listener-sa-policy"
+  name   = "${var.eks_foxglove_namespace}-inbox-listener-sa-policy"
   path   = "/"
   policy = data.aws_iam_policy_document.inbox_listener_policy_document.json
 }

primary-site/aws/modules/iam/stream-service.tf

@@ -5,7 +5,7 @@ data "aws_iam_policy_document" "stream_service_policy_document" {
       "s3:ListBucket"
     ]
     resources = [var.lake_bucket_arn]
-    effect = "Allow"
+    effect    = "Allow"
   }
   statement {
     actions = [
@@ -24,7 +24,7 @@ data "aws_iam_policy_document" "stream_service_policy_document" {
 }
 
 resource "aws_iam_policy" "stream_service_policy" {
-  name = "${var.eks_foxglove_namespace}-stream-service-sa-policy"
+  name   = "${var.eks_foxglove_namespace}-stream-service-sa-policy"
   path   = "/"
   policy = data.aws_iam_policy_document.stream_service_policy_document.json
 }

primary-site/aws/modules/s3/main.tf

@@ -9,8 +9,32 @@ resource "aws_s3_bucket_lifecycle_configuration" "bucket_lifecycle" {
     id     = "${var.bucket_name}-delete-incomplete-uploads-rule"
     status = "Enabled"
 
+    # At least one filter rule is required by Terraform.
+    # Default to an empty prefix to include everything.
+    filter {
+      prefix = ""
+    }
+
     abort_incomplete_multipart_upload {
       days_after_initiation = var.abort_incomplete_multipart_upload_days
     }
   }
+
+  # Failed imports are cleaned up in the control plane after a year, this rule will
+  # remove their backing objects the following day.
+  #
+  # This rule is only required for the inbox bucket, but it will have no impact on
+  # the lake bucket so it is fine here.
+  rule {
+    id     = "${var.bucket_name}-delete-old-quarantined-files"
+    status = "Enabled"
+
+    filter {
+      prefix = "_quarantine/"
+    }
+
+    expiration {
+      days = 366
+    }
+  }
 }

primary-site/aws/modules/sns/main.tf

@@ -81,7 +81,7 @@ data "aws_iam_policy_document" "sqs_dlq_policy" {
 
 resource "aws_sqs_queue_policy" "allow_publish_from_sns" {
   queue_url = aws_sqs_queue.dlq.id
-  policy = data.aws_iam_policy_document.sqs_dlq_policy.json
+  policy    = data.aws_iam_policy_document.sqs_dlq_policy.json
 }
 
 resource "aws_sns_topic_subscription" "webhook" {

primary-site/aws/provider.tf

@@ -5,5 +5,5 @@ terraform {
 }
 
 provider "aws" {
-  region  = "us-east-1"
+  region = "us-east-1"
 }

primary-site/azure/modules/iam/main.tf

@@ -8,8 +8,8 @@ resource "azuread_application" "primary_site" {
 }
 
 resource "azuread_service_principal" "iam_principal" {
-  client_id      = azuread_application.primary_site.client_id
-  owners         = [data.azuread_client_config.current.object_id]
+  client_id = azuread_application.primary_site.client_id
+  owners    = [data.azuread_client_config.current.object_id]
 }
 
 resource "azuread_service_principal_password" "iam_principal" {

primary-site/azure/modules/storage/main.tf

@@ -1,4 +1,7 @@
 ## ----- Storage -----
+locals {
+  quarantine_cutoff_days = 366
+}
 
 resource "azurerm_storage_account" "storage" {
   name                     = var.storage_account_name
@@ -38,6 +41,25 @@ resource "azurerm_storage_management_policy" "storage_policy" {
       }
     }
   }
+  rule {
+    name    = "deleteOldQuarantinedFiles"
+    enabled = true
+    filters {
+      prefix_match = ["inbox/_quarantine/"]
+      blob_types   = ["blockBlob"]
+    }
+    actions {
+      base_blob {
+        delete_after_days_since_modification_greater_than = local.quarantine_cutoff_days
+      }
+      snapshot {
+        delete_after_days_since_creation_greater_than = local.quarantine_cutoff_days
+      }
+      version {
+        delete_after_days_since_creation = local.quarantine_cutoff_days
+      }
+    }
+  }
 }
 
 resource "azurerm_storage_container" "inbox" {

primary-site/gcp/modules/pubsub/main.tf

@@ -14,13 +14,13 @@ module "inbox_notifications" {
   topic      = "${var.bucket_name}-notifications"
   push_subscriptions = [
     {
-      name                  = "${var.bucket_name}-push-sub"
-      push_endpoint         = var.inbox_notification_endpoint
+      name          = "${var.bucket_name}-push-sub"
+      push_endpoint = var.inbox_notification_endpoint
 
       # To avoid the error where the push_subscriptions "for_each" map includes keys derived
       # from resource attributes that cannot be determined until apply, we "hard-code" the dead
       # letter topic name here. This is why `depends_on` is needed above.
-      dead_letter_topic     = "projects/${var.gcp_project}/topics/${var.bucket_name}-notifications-dlq"
+      dead_letter_topic = "projects/${var.gcp_project}/topics/${var.bucket_name}-notifications-dlq"
 
       x-goog-version        = "v1"
       ack_deadline_seconds  = 600

primary-site/gcp/modules/storage/main.tf

@@ -27,8 +27,8 @@ resource "google_storage_bucket" "bucket" {
     }
   }
 
-  // Deleting `tmp/` prefixed objects is required for the lake bucket only, but has
-  // no impact on the inbox bucket
+  # Deleting `tmp/` prefixed objects is required for the lake bucket only, but has
+  # no impact on the inbox bucket
   lifecycle_rule {
     action {
       type = "Delete"
@@ -39,4 +39,20 @@ resource "google_storage_bucket" "bucket" {
       age            = 1
     }
   }
+
+  # Failed imports are cleaned up in the control plane after a year, this rule will
+  # remove their backing objects the following day.
+  #
+  # This rule is only required for the inbox bucket to the inbox bucket, but will
+  # have no impact on the lake bucket.
+  lifecycle_rule {
+    action = {
+      type = "Delete"
+    }
+    condition {
+      with_state     = "LIVE"
+      matches_prefix = ["_quarantine/"]
+      age            = 366
+    }
+  }
 }