NRL-2002 Dynamo Export

lambdas/dynamo_export_poll/dynamo_export_poll.py

@@ -0,0 +1,31 @@
+import boto3
+from botocore.config import Config
+
+ddb = boto3.client(
+    "dynamodb",
+    config=Config(connect_timeout=5, read_timeout=5),
+)
+
+
+def lambda_handler(event, _):
+    completed = []
+    for arn in event["export_arns"]:
+        response = ddb.describe_export(ExportArn=arn)
+        if response["ExportDescription"]["ExportStatus"] == "FAILED":
+            return {
+                "status": "FAILED",
+                "export_to_time": event["export_to_time"],
+                "export_arns": event["export_arns"],
+                "export_type": event["export_type"],
+            }
+
+        completed.append(response["ExportDescription"]["ExportStatus"])
+
+    status = "COMPLETED" if all(s == "COMPLETED" for s in completed) else "IN_PROGRESS"
+
+    return {
+        "status": status,
+        "export_to_time": event["export_to_time"],
+        "export_arns": event["export_arns"],
+        "export_type": event["export_type"],
+    }

lambdas/dynamo_export_trigger/dynamo_export_trigger.py

@@ -0,0 +1,83 @@
+import os
+from datetime import datetime, timedelta, timezone
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+
+bucket = os.environ["BUCKET"]
+ddb_table_arn = os.environ["DDB_TABLE_ARN"]
+kms_key = os.environ["KMS_KEY"]
+env = os.environ["ENVIRONMENT"]
+ddb_table_name = os.environ["DDB_TABLE_NAME"]
+
+SSM_PARAM = "/exports/DynamoExportRuntime"
+
+ddb_client = boto3.client(
+    "dynamodb",
+    config=Config(connect_timeout=5, read_timeout=5),
+)
+ssm = boto3.client(
+    "ssm",
+    config=Config(connect_timeout=5, read_timeout=5),
+)
+
+
+def lambda_handler(_, __):
+    to_time = datetime.now(timezone.utc).replace(microsecond=0, second=0, minute=0)
+    export_arns = []
+
+    try:
+        from_time = ssm.get_parameter(Name=SSM_PARAM)["Parameter"]["Value"]
+        from_time = datetime.fromisoformat(from_time).replace(
+            microsecond=0, second=0, minute=0
+        )
+
+        # Handle exports longer than 24 hours by splitting into multiple exports
+        earliest_pitr = ddb_client.describe_continuous_backups(
+            TableName=ddb_table_name
+        )["ContinuousBackupsDescription"]["PointInTimeRecoveryDescription"][
+            "EarliestRestorableDateTime"
+        ]
+        from_time = max(from_time, earliest_pitr)
+        days_difference = (to_time - from_time).days + 1
+        from_times = [from_time + timedelta(days=i) for i in range(days_difference)]
+
+        for base_time in from_times:
+            end_time = min(base_time + timedelta(days=1), to_time)
+            if end_time == base_time:
+                continue
+            response = ddb_client.export_table_to_point_in_time(
+                TableArn=ddb_table_arn,
+                S3Bucket=bucket,
+                S3SseAlgorithm="KMS",
+                S3SseKmsKeyId=kms_key,
+                ExportFormat="DYNAMODB_JSON",
+                ExportType="INCREMENTAL_EXPORT",
+                IncrementalExportSpecification={
+                    "ExportFromTime": base_time,
+                    "ExportToTime": end_time,
+                    "ExportViewType": "NEW_AND_OLD_IMAGES",
+                },
+            )
+            export_arns.append(response["ExportDescription"]["ExportArn"])
+            export_type = response["ExportDescription"]["ExportType"]
+    except ClientError as e:
+        if e.response["Error"]["Code"] != "ParameterNotFound":
+            raise
+        response = ddb_client.export_table_to_point_in_time(
+            TableArn=ddb_table_arn,
+            S3Bucket=bucket,
+            S3SseAlgorithm="KMS",
+            S3SseKmsKeyId=kms_key,
+            ExportFormat="DYNAMODB_JSON",
+            ExportType="FULL_EXPORT",
+        )
+        export_arns.append(response["ExportDescription"]["ExportArn"])
+        export_type = response["ExportDescription"]["ExportType"]
+
+    return {
+        "export_to_time": to_time.isoformat(),
+        "export_arns": export_arns,
+        "export_type": export_type,
+    }

lambdas/ssm_put_param/ssm_put_param.py

@@ -0,0 +1,18 @@
+import boto3
+from botocore.config import Config
+
+ssm = boto3.client(
+    "ssm",
+    config=Config(connect_timeout=5, read_timeout=5),
+)
+
+
+def lambda_handler(event, _):
+    param_name = "/exports/DynamoExportRuntime"
+    param_value = event["export_to_time"]
+    ssm.put_parameter(Name=param_name, Value=param_value, Type="String", Overwrite=True)
+    return {
+        "to_time": param_value,
+        "export_arns": event["export_arns"],
+        "export_type": event["export_type"],
+    }

terraform/account-wide-infrastructure/dev/dynamo_export.tf

@@ -0,0 +1,6 @@
+module "dynamo_export" {
+  source             = "./modules/dynamo_export"
+  name_prefix        = "nhsd-nrlf--dev"
+  environment        = "dev"
+  pointer_table_name = module.dev-pointers-table.table_name
+}

terraform/account-wide-infrastructure/modules/dynamo_export/dynamo_export_step_function_definition.json

@@ -0,0 +1,87 @@
+{
+  "Comment": "execute lambdas",
+  "StartAt": "DynamoExport",
+  "States": {
+    "DynamoExport": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "OutputPath": "$.Payload",
+      "Parameters": {
+        "FunctionName": "${lambda_export_trigger_function_name}",
+        "Payload.$": "$"
+      },
+      "Next": "DynamoExportStatusCheck"
+    },
+    "DynamoExportStatusCheck": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "OutputPath": "$.Payload",
+      "Parameters": {
+        "FunctionName": "${lambda_export_poll_function_name}",
+        "Payload.$": "$"
+      },
+      "Next": "Choice"
+    },
+    "Choice": {
+      "Type": "Choice",
+      "Choices": [
+        {
+          "Variable": "$.status",
+          "StringEquals": "COMPLETED",
+          "Next": "SSMPut"
+        },
+        {
+          "Variable": "$.status",
+          "StringEquals": "IN_PROGRESS",
+          "Next": "WaitState"
+        },
+        {
+          "Variable": "$.status",
+          "StringEquals": "FAILED",
+          "Next": "ExportFailure"
+        }
+      ],
+      "Default": "FailState"
+    },
+    "FailState": {
+      "Type": "Fail",
+      "Error": "UnhandledStatus",
+      "Cause": "Status not recognised"
+    },
+    "ExportFailure": {
+      "Type": "Fail",
+      "Error": "DynamoExportFailed",
+      "Cause": "DynamoDB Export Failed"
+    },
+    "WaitState": {
+      "Type": "Wait",
+      "Seconds": 120,
+      "Next": "DynamoExportStatusCheck"
+    },
+    "SSMPut": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::lambda:invoke",
+      "OutputPath": "$.Payload",
+      "Parameters": {
+        "FunctionName": "${lambda_ssm_put_param_function_name}",
+        "Payload.$": "$"
+      },
+      "Next": "GlueJobTrigger"
+    },
+    "GlueJobTrigger": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::glue:startJobRun.sync",
+      "Parameters": {
+        "JobName": "${glue_job_name}",
+        "Arguments": {
+          "--SOURCE_BUCKET": "${dynamo_export_s3_bucket_name}",
+          "--TARGET_BUCKET": "${dynamo_export_processed_s3_bucket_name}",
+          "--DDB_TABLE_ARN": "${ddb_table_arn}",
+          "--GLUE_CRAWLER_NAME": "${glue_crawler_name}",
+          "--EXPORT_TYPE.$": "$.export_type"
+        }
+      },
+      "End": true
+    }
+  }
+}

terraform/account-wide-infrastructure/modules/dynamo_export/dynamo_output_s3.tf

@@ -0,0 +1,77 @@
+resource "aws_s3_bucket" "dynamodb_output" {
+  bucket = "${var.name_prefix}-${var.environment}-dynamo-output-bucket"
+}
+
+# May need to restrict access to specific IAM roles/principals in future, but helps with testing for now.
+data "aws_iam_policy_document" "dynamodb_output" {
+  statement {
+    sid     = "HTTPSOnly"
+    effect  = "Deny"
+    actions = ["s3:*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    resources = [
+      aws_s3_bucket.dynamodb_output.arn,
+      "${aws_s3_bucket.dynamodb_output.arn}/*"
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
+resource "aws_s3_bucket_policy" "dynamodb_output" {
+  bucket = aws_s3_bucket.dynamodb_output.id
+  policy = data.aws_iam_policy_document.dynamodb_output.json
+}
+
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "dynamodb_output" {
+  bucket = aws_s3_bucket.dynamodb_output.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      kms_master_key_id = aws_kms_key.dynamo.arn
+      sse_algorithm     = "aws:kms"
+    }
+  }
+}
+
+
+resource "aws_s3_bucket_public_access_block" "dynamodb_output_public_access_block" {
+  bucket = aws_s3_bucket.dynamodb_output.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "dynamodb_output_lifecycle" {
+  bucket = aws_s3_bucket.dynamodb_output.id
+
+
+  rule {
+    id     = "object-auto-delete-rule"
+    status = "Enabled"
+    filter {}
+
+    expiration {
+      days = 2
+    }
+  }
+}
+
+resource "aws_s3_bucket_versioning" "dynamodb_output_versioning" {
+  bucket = aws_s3_bucket.dynamodb_output.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}

terraform/account-wide-infrastructure/modules/dynamo_export/dynamo_processed_output_s3.tf

@@ -0,0 +1,76 @@
+resource "aws_s3_bucket" "dynamodb_output_processed" {
+  bucket = "${var.name_prefix}-${var.environment}-dynamo-output-processed-bucket"
+}
+
+data "aws_iam_policy_document" "dynamodb_output_processed" {
+  statement {
+    sid     = "HTTPSOnly"
+    effect  = "Deny"
+    actions = ["s3:*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    resources = [
+      aws_s3_bucket.dynamodb_output_processed.arn,
+      "${aws_s3_bucket.dynamodb_output_processed.arn}/*"
+    ]
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}
+
+resource "aws_s3_bucket_policy" "dynamodb_output_processed" {
+  bucket = aws_s3_bucket.dynamodb_output_processed.id
+  policy = data.aws_iam_policy_document.dynamodb_output_processed.json
+}
+
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "dynamodb_output_processed" {
+  bucket = aws_s3_bucket.dynamodb_output_processed.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      kms_master_key_id = aws_kms_key.dynamo_processed.arn
+      sse_algorithm     = "aws:kms"
+    }
+  }
+}
+
+
+resource "aws_s3_bucket_public_access_block" "dynamodb_output_processed_public_access_block" {
+  bucket = aws_s3_bucket.dynamodb_output_processed.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "dynamodb_output_processed_lifecycle" {
+  bucket = aws_s3_bucket.dynamodb_output_processed.id
+
+
+  rule {
+    id     = "object-auto-delete-rule"
+    status = "Enabled"
+    filter {}
+
+    expiration {
+      days = 3 * 365
+    }
+  }
+}
+
+resource "aws_s3_bucket_versioning" "dynamodb_output_processed_versioning" {
+  bucket = aws_s3_bucket.dynamodb_output_processed.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}