Skip to content

S3 to Azure backup #236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Stretch96 wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

S3 to Azure backup #236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_cloudwatch_event_rule.infrastructure_ecs_cluster_service_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_rule.infrastructure_rds_s3_backups_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_rule.infrastructure_rds_tooling_image_build_trigger_codebuild](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_rule.infrastructure_s3_to_azure_backup_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_target.ecr_scan_event_target](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.ecs_cluster_infrastructure_ecs_asg_diff_metric_1_min_cron](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.ecs_cluster_infrastructure_instance_refresh](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
Expand All @@ -80,6 +81,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_cloudwatch_event_target.infrastructure_ecs_cluster_service_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.infrastructure_rds_s3_backups_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.infrastructure_rds_tooling_image_build_trigger_codebuild](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_event_target.infrastructure_s3_to_azure_backup_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_cloudwatch_log_group.ecs_cluster_infrastructure_draining_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
| [aws_cloudwatch_log_group.ecs_cluster_infrastructure_ecs_asg_diff_metric_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
| [aws_cloudwatch_log_group.ecs_cluster_infrastructure_instance_refresh_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
Expand Down Expand Up @@ -193,7 +195,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_iam_policy.infrastructure_ecs_cluster_service_task_ssm_create_channels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_ecs_run_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_s3_backups_task_s3_list](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_s3_backups_task_s3_write](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_tooling_image_codebuild_allow_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
Expand All @@ -207,6 +209,9 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_iam_policy.infrastructure_rds_tooling_task_execution_get_secret_value](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_tooling_task_kms_encrypt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_rds_tooling_task_ssm_create_channels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_s3_to_azure_backup_cloudwatch_schedule_ecs_run_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_s3_to_azure_backup_cloudwatch_schedule_pass_role_tooling_task_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_policy.infrastructure_s3_to_azure_backup_task_s3_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_role.ecs_cluster_infrastructure_draining_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.ecs_cluster_infrastructure_ecs_asg_diff_metric_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.ecs_cluster_infrastructure_instance_refresh_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
Expand All @@ -227,6 +232,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_iam_role.infrastructure_rds_tooling_image_codebuild](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.infrastructure_rds_tooling_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.infrastructure_rds_tooling_task_execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.infrastructure_s3_to_azure_backup_cloudwatch_schedule](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role.infrastructure_vpc_flow_logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy.infrastructure_vpc_flow_logs_allow_cloudwatch_rw](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy_attachment.ecs_cluster_infrastructure_draining_ecs_container_instance_state_update_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
Expand Down Expand Up @@ -283,7 +289,7 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_iam_role_policy_attachment.infrastructure_ecs_cluster_service_task_ssm_create_channels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_monitoring](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_ecs_run_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_task_s3_list](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_task_s3_write](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_tooling_image_codebuild_allow_builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
Expand All @@ -297,6 +303,9 @@ This project creates and manages resources within an AWS account for infrastruct
| [aws_iam_role_policy_attachment.infrastructure_rds_tooling_task_execution_get_secret_value](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_tooling_task_kms_encrypt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_rds_tooling_task_ssm_create_channels](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_s3_to_azure_backup_cloudwatch_schedule_ecs_run_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_s3_to_azure_backup_cloudwatch_schedule_pass_role_tooling_task_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.infrastructure_s3_to_azure_backup_task_s3_write](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_instance.infrastructure_bastion](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance) | resource |
| [aws_internet_gateway.infrastructure_public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/internet_gateway) | resource |
| [aws_kms_alias.custom_s3_buckets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource |
Expand Down Expand Up @@ -578,6 +587,11 @@ This project creates and manages resources within an AWS account for infrastruct
| <a name="input_infrastructure_rds_backup_to_s3_cron_expression"></a> [infrastructure\_rds\_backup\_to\_s3\_cron\_expression](#input\_infrastructure\_rds\_backup\_to\_s3\_cron\_expression) | Cron expression for when to trigger the SQL backups to S3 | `string` | n/a | yes |
| <a name="input_infrastructure_rds_backup_to_s3_retention"></a> [infrastructure\_rds\_backup\_to\_s3\_retention](#input\_infrastructure\_rds\_backup\_to\_s3\_retention) | Retention in days to keep the S3 SQL backups | `number` | n/a | yes |
| <a name="input_infrastructure_rds_defaults"></a> [infrastructure\_rds\_defaults](#input\_infrastructure\_rds\_defaults) | Default values for RDSs | <pre>object({<br/> type = optional(string, null)<br/> engine = optional(string, null)<br/> engine_version = optional(string, null)<br/> parameters = optional(map(string), null)<br/> instance_class = optional(string, null)<br/> allocated_storage = optional(number, null)<br/> storage_type = optional(string, null)<br/> dedicated_kms_key = optional(bool, null)<br/> dedicated_kms_key_policy_statements = optional(string, null)<br/> iops = optional(number, null)<br/> storage_throughput = optional(number, null)<br/> multi_az = optional(bool, null)<br/> monitoring_interval = optional(number, null)<br/> cloudwatch_logs_export_types = optional(list(string), null)<br/> cluster_instance_count = optional(number, null)<br/> cluster_serverlessv2_min_capacity = optional(number, null)<br/> cluster_serverlessv2_max_capacity = optional(number, null)<br/> })</pre> | n/a | yes |
| <a name="input_infrastructure_s3_to_azure_backup"></a> [infrastructure\_s3\_to\_azure\_backup](#input\_infrastructure\_s3\_to\_azure\_backup) | List of objects, defining S3 bucket name and Azure container name, along with Azure credentials<br/> [<br/> {<br/> s3\_bucket\_name: The s3 bucket name (source)<br/> azure\_container\_name: The Azure container name (target)<br/> }<br/> ] | <pre>list(object({<br/> s3_bucket_name = string<br/> azure_container_name = string<br/> }))</pre> | n/a | yes |
| <a name="input_infrastructure_s3_to_azure_backup_azure_spa_application_id"></a> [infrastructure\_s3\_to\_azure\_backup\_azure\_spa\_application\_id](#input\_infrastructure\_s3\_to\_azure\_backup\_azure\_spa\_application\_id) | Azure service principle app (spa) application id for the S3 to Azure backups | `string` | n/a | yes |
| <a name="input_infrastructure_s3_to_azure_backup_azure_spa_client_secret"></a> [infrastructure\_s3\_to\_azure\_backup\_azure\_spa\_client\_secret](#input\_infrastructure\_s3\_to\_azure\_backup\_azure\_spa\_client\_secret) | Azure service principle app (spa) client secret for the S3 to Azure backups | `string` | n/a | yes |
| <a name="input_infrastructure_s3_to_azure_backup_azure_tenant_id"></a> [infrastructure\_s3\_to\_azure\_backup\_azure\_tenant\_id](#input\_infrastructure\_s3\_to\_azure\_backup\_azure\_tenant\_id) | Azure tenant id for the S3 to Azure backups | `string` | n/a | yes |
| <a name="input_infrastructure_s3_to_azure_backup_cron_expression"></a> [infrastructure\_s3\_to\_azure\_backup\_cron\_expression](#input\_infrastructure\_s3\_to\_azure\_backup\_cron\_expression) | Cron expression for when to trigger the S3 to Azure backup | `string` | n/a | yes |
| <a name="input_infrastructure_vpc"></a> [infrastructure\_vpc](#input\_infrastructure\_vpc) | Enable infrastructure VPC | `bool` | n/a | yes |
| <a name="input_infrastructure_vpc_assign_generated_ipv6_cidr_block"></a> [infrastructure\_vpc\_assign\_generated\_ipv6\_cidr\_block](#input\_infrastructure\_vpc\_assign\_generated\_ipv6\_cidr\_block) | Assign generated IPv6 CIDR block on infrastructure VPC | `bool` | n/a | yes |
| <a name="input_infrastructure_vpc_cidr_block"></a> [infrastructure\_vpc\_cidr\_block](#input\_infrastructure\_vpc\_cidr\_block) | Infrastructure VPC CIDR block | `string` | n/a | yes |
Expand Down
16 changes: 15 additions & 1 deletion locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ locals {
enable_infrastructure_rds_backup_to_s3 = var.enable_infrastructure_rds_backup_to_s3
infrastructure_rds_backup_to_s3_cron_expression = var.infrastructure_rds_backup_to_s3_cron_expression
infrastructure_rds_backup_to_s3_retention = var.infrastructure_rds_backup_to_s3_retention
enable_infrastructure_rds_tooling = length(var.infrastructure_rds) > 0
enable_infrastructure_rds_tooling = length(local.infrastructure_rds) > 0 || length(local.infrastructure_s3_to_azure_backup) > 0
infrastructure_rds_tooling_ecs_cluster_name = "${local.resource_prefix}-infrastructure-rds-tooling"

infrastructure_elasticache_defaults = var.infrastructure_elasticache_defaults
Expand All @@ -272,6 +272,20 @@ locals {
enable_cloudformatian_s3_template_store = var.enable_cloudformatian_s3_template_store != null ? var.enable_cloudformatian_s3_template_store : false
custom_cloudformation_stacks = var.custom_cloudformation_stacks

infrastructure_s3_to_azure_backup = var.infrastructure_s3_to_azure_backup
infrastructure_s3_to_azure_backup_command = join(
" && ",
[
for obj in local.infrastructure_s3_to_azure_backup : "s3-to-azure -d \"${obj["s3_bucket_name"]}\" -d \"${obj["azure_container_name"]}\""
Copy link
Contributor

@DrizzlyOwl DrizzlyOwl Apr 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be -s for source and -d for destination

])
infrastructure_s3_to_azure_backup_command_s3_buckets = toset([
for obj in local.infrastructure_s3_to_azure_backup : obj["s3_bucket_name"]
])
infrastructure_s3_to_azure_backup_cron_expression = var.infrastructure_s3_to_azure_backup_cron_expression
infrastructure_s3_to_azure_backup_azure_tenant_id = var.infrastructure_s3_to_azure_backup_azure_tenant_id
infrastructure_s3_to_azure_backup_azure_spa_application_id = var.infrastructure_s3_to_azure_backup_azure_spa_application_id
infrastructure_s3_to_azure_backup_azure_spa_client_secret = var.infrastructure_s3_to_azure_backup_azure_spa_client_secret

s3_object_presign = local.enable_cloudformatian_s3_template_store ? toset([
for k, v in local.custom_cloudformation_stacks : "${aws_s3_bucket.cloudformation_custom_stack_template_store[0].id}/${v["s3_template_store_key"]}" if v["s3_template_store_key"] != null
]) : []
Expand Down
12 changes: 6 additions & 6 deletions rds-infrastructure-s3-backups-scheduled-task.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ resource "aws_iam_role_policy_attachment" "infrastructure_rds_s3_backups_cloudwa
policy_arn = aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_ecs_run_task[each.key].arn
}

resource "aws_iam_policy" "infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role" {
resource "aws_iam_policy" "infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles" {
for_each = local.enable_infrastructure_rds_backup_to_s3 ? local.infrastructure_rds : {}

name = "${local.resource_prefix}-${substr(sha512("rds-s3-backups-cloudwatch-schedule-${each.key}-pass-role-execution-role"), 0, 6)}"
description = "${local.resource_prefix}-rds-s3-backups-cloudwatch-schedule-${each.key}-pass-role-execution-role"
name = "${local.resource_prefix}-${substr(sha512("rds-s3-backups-cloudwatch-schedule-${each.key}-pass-role-tooling-task-roles"), 0, 6)}"
description = "${local.resource_prefix}-rds-s3-backups-cloudwatch-schedule-${each.key}-pass-role-tooling-task-roles"
policy = templatefile(
"${path.root}/policies/pass-role.json.tpl",
{
Expand All @@ -41,11 +41,11 @@ resource "aws_iam_policy" "infrastructure_rds_s3_backups_cloudwatch_schedule_pas
)
}

resource "aws_iam_role_policy_attachment" "infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role" {
resource "aws_iam_role_policy_attachment" "infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles" {
for_each = local.enable_infrastructure_rds_backup_to_s3 ? local.infrastructure_rds : {}

role = aws_iam_role.infrastructure_rds_s3_backups_cloudwatch_schedule[each.key].name
policy_arn = aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role[each.key].arn
policy_arn = aws_iam_policy.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles[each.key].arn
}

resource "aws_cloudwatch_event_rule" "infrastructure_rds_s3_backups_scheduled_task" {
Expand Down Expand Up @@ -96,6 +96,6 @@ resource "aws_cloudwatch_event_target" "infrastructure_rds_s3_backups_scheduled_

depends_on = [
aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_ecs_run_task,
aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role,
aws_iam_role_policy_attachment.infrastructure_rds_s3_backups_cloudwatch_schedule_pass_role_tooling_task_roles,
]
}
Loading
Loading