Multiple AWS SM Secrets as inputs and RDS Fixes (#52)

* Initial commit

* Adding option for DB Identifier

* Typo

* Making proxy policy more flexible

* Typo 2

* Ignoring policy changes

* Trying a more direct approach for db_proxy_target

* lowering-var

* Testing with lifecycle block

* Debugging proxy recreation

* Debugging proxy recreation 2

* v3

Author: LeoDiazL

README.md

@@ -128,7 +128,7 @@ The following inputs can be used as `step.with` keys
 #### **Secrets and Environment Variables Inputs**
 | Name             | Type    | Description - Check note about [**environment variables**](#environment-variables). |
 |------------------|---------|------------------------------------|
-| `env_aws_secret` | String | Secret name to pull environment variables from AWS Secret Manager. |
+| `env_aws_secret` | String | Secret name to pull environment variables from AWS Secret Manager. Accepts comma separated list of secrets. |
 | `env_repo` | String | `.env` file containing environment variables to be used with the app. Name defaults to `repo_env`. |
 | `env_ghs` | String | `.env` file to be used with the app. This is the name of the [Github secret](https://docs.github.com/es/actions/security-guides/encrypted-secrets). |
 | `env_ghv` | String | `.env` file to be used with the app. This is the name of the [Github variables](https://docs.github.com/en/actions/learn-github-actions/variables). |
@@ -225,6 +225,7 @@ The following inputs can be used as `step.with` keys
 |------------------|---------|------------------------------------|
 | `aws_rds_db_enable`| Boolean | Set to `true` to enable an RDS DB. |
 | `aws_rds_db_proxy`| Boolean | Set to `true` to add a RDS DB Proxy. |
+| `aws_rds_db_identifier`| String | Database identifier that will appear in the AWS Console. Defaults to `aws_resource_identifier` if none set. |
 | `aws_rds_db_name`| String | The name of the database to create when the DB instance is created. If this parameter is not specified, no database is created in the DB instance. |
 | `aws_rds_db_user`| String | Username for the db. Defaults to `dbuser`. |
 | `aws_rds_db_engine`| String | Which Database engine to use. Defaults to `postgres`. |
@@ -509,6 +510,25 @@ The Aurora available environment variables are:
 | `AURORA_CLUSTER_ENGINE_VERSION_ACTUAL` | The running version of the cluster database |
 | `AURORA_CLUSTER_HOSTED_ZONE_ID`| The Route53 Hosted Zone ID of the endpoint |
 
+### Stored secret in AWS Secrets Manager
+In order to be flexible, the following variables will be used to store DB related info in AWS Secretes Manager
+
+`username`
+`password`
+`host`
+`port`
+`database`
+`engine`
+`engine_version`
+`DB_USER`
+`DB_USERNAME`
+`DB_PASSWORD`
+`DB_HOST`
+`DB_PORT`
+`DB_NAME`
+`DB_ENGINE`
+`DB_ENGINE_VERSION`
+
 ### AWS Root Certs
 The AWS root certificate is downloaded and accessible via the `rds-combined-ca-bundle.pem` file in root of your app repo/directory.
 

action.yaml

@@ -119,7 +119,7 @@ inputs:
 
   # ENV files
   env_aws_secret:
-    description: 'Secret name to pull env variables from AWS Secret Manager'
+    description: 'Secret name to pull env variables from AWS Secret Manager, could be a comma separated list, read in order. Expected JSON content.'
     required: false
   env_repo:
     description: 'File containing environment variables to be used with the app'
@@ -311,6 +311,9 @@ inputs:
   aws_rds_db_proxy:
     description: 'Set to true to add a RDS DB Proxy'
     required: false
+  aws_rds_db_identifier:
+    description: 'Database identifier that will appear in the AWS Console. Defaults to aws_resource_identifier if none set.'
+    required: false
   aws_rds_db_name:
     description: 'The name of the database to create when the DB instance is created.'
     required: false
@@ -902,6 +905,7 @@ runs:
         # AWS RDS
         AWS_RDS_DB_ENABLE: ${{ inputs.aws_rds_db_enable }}
         AWS_RDS_DB_PROXY: ${{ inputs.aws_rds_db_proxy }}
+        AWS_RDS_DB_IDENTIFIER: ${{ inputs.aws_rds_db_identifier }}
         AWS_RDS_DB_NAME: ${{ inputs.aws_rds_db_name }}
         AWS_RDS_DB_USER: ${{ inputs.aws_rds_db_user }}
         AWS_RDS_DB_ENGINE: ${{ inputs.aws_rds_db_engine }}

operations/_scripts/generate/generate_vars_terraform.sh

@@ -162,6 +162,7 @@ fi
 if [[ $(alpha_only "$AWS_RDS_DB_ENABLE") == true ]]; then
   aws_rds_db_enable=$(generate_var aws_rds_db_enable $AWS_RDS_DB_ENABLE)
   aws_rds_db_proxy=$(generate_var aws_rds_db_proxy $AWS_RDS_DB_PROXY)
+  aws_rds_db_identifier=$(generate_var aws_rds_db_identifier $AWS_RDS_DB_IDENTIFIER)
   aws_rds_db_name=$(generate_var aws_rds_db_name $AWS_RDS_DB_NAME)
   aws_rds_db_user=$(generate_var aws_rds_db_user $AWS_RDS_DB_USER)
   aws_rds_db_engine=$(generate_var aws_rds_db_engine $AWS_RDS_DB_ENGINE)
@@ -420,6 +421,7 @@ $aws_efs_additional_tags
 #-- RDS --#
 $aws_rds_db_enable
 $aws_rds_db_proxy
+$aws_rds_db_identifier
 $aws_rds_db_name
 $aws_rds_db_user
 $aws_rds_db_engine

operations/deployment/terraform/aws/aws_variables.tf

@@ -396,6 +396,12 @@ variable "aws_rds_db_proxy" {
   default     = false
 }
 
+variable "aws_rds_db_identifier" {
+  type        = string
+  description = "Database identifier that will appear in the AWS Console. Defaults to aws_resource_identifier if none set."
+  default     = ""
+}
+
 variable "aws_rds_db_name" {
   type        = string
   description = "The name of the database to create when the DB instance is created. If this parameter is not specified, no database is created in the DB instance."

operations/deployment/terraform/aws/bitovi_main.tf

@@ -134,6 +134,7 @@ module "rds" {
   # RDS
   aws_rds_db_name                        = var.aws_rds_db_name
   aws_rds_db_user                        = var.aws_rds_db_user
+  aws_rds_db_identifier                  = var.aws_rds_db_identifier != "" ? var.aws_rds_db_identifier : lower(var.aws_resource_identifier)
   aws_rds_db_engine                      = var.aws_rds_db_engine
   aws_rds_db_engine_version              = var.aws_rds_db_engine_version
   aws_rds_db_security_group_name         = var.aws_rds_db_security_group_name

operations/deployment/terraform/ecr/bitovi_main.tf

@@ -42,7 +42,7 @@ locals {
   }
   default_tags = merge(local.aws_tags, jsondecode(var.aws_additional_tags))
   # Module tagging
-  ecr_tags    = merge(local.default_tags,jsondecode(var.aws_ecr_additional_tags))
+  ecr_tags     = merge(local.default_tags,jsondecode(var.aws_ecr_additional_tags))
 }
 
 output "ecr_repository_arn" {

operations/deployment/terraform/modules/aws/aurora/aws_aurora.tf

@@ -136,13 +136,19 @@ resource "aws_secretsmanager_secret_version" "database_credentials_sm_secret_ver
   secret_string = jsonencode({
    username          = sensitive(module.aurora_cluster.cluster_master_username)
    password          = sensitive(module.aurora_cluster.cluster_master_password)
-   DB_ENGINE         = sensitive(local.dba_engine)
-   DB_ENGINE_VERSION = sensitive(module.aurora_cluster.cluster_engine_version_actual)
+   host              = sensitive(module.aurora_cluster.cluster_endpoint)
+   port              = sensitive(module.aurora_cluster.cluster_port)
+   database          = sensitive(module.aurora_cluster.cluster_database_name == null ? "" : module.aurora_cluster.cluster_database_name)
+   engine            = sensitive(local.dba_engine)
+   engine_version    = sensitive(module.aurora_cluster.cluster_engine_version_actual)
    DB_USER           = sensitive(module.aurora_cluster.cluster_master_username)
+   DB_USERNAME       = sensitive(module.aurora_cluster.cluster_master_username)
    DB_PASSWORD       = sensitive(module.aurora_cluster.cluster_master_password)
-   DB_NAME           = sensitive(module.aurora_cluster.cluster_database_name == null ? "" : module.aurora_cluster.cluster_database_name)
-   DB_PORT           = sensitive(module.aurora_cluster.cluster_port)
    DB_HOST           = sensitive(module.aurora_cluster.cluster_endpoint)
+   DB_PORT           = sensitive(module.aurora_cluster.cluster_port)
+   DB_NAME           = sensitive(module.aurora_cluster.cluster_database_name == null ? "" : module.aurora_cluster.cluster_database_name)
+   DB_ENGINE         = sensitive(local.dba_engine)
+   DB_ENGINE_VERSION = sensitive(module.aurora_cluster.cluster_engine_version_actual)
   })
 }
 

operations/deployment/terraform/modules/aws/db_proxy/aws_db_proxy.tf

@@ -81,7 +81,10 @@ resource "aws_db_proxy_target" "db_instance" {
   db_instance_identifier = data.aws_db_instance.db[0].id
   db_proxy_name          = aws_db_proxy.rds_proxy[0].name
   target_group_name      = aws_db_proxy_default_target_group.default.name
-
+  lifecycle {
+    ignore_changes = [ db_instance_identifier ]
+    replace_triggered_by = [ data.aws_db_instance.db ]
+  }
   depends_on = [ aws_db_proxy.rds_proxy ]
 }
 
@@ -90,7 +93,10 @@ resource "aws_db_proxy_target" "db_cluster" {
   db_cluster_identifier  = data.aws_rds_cluster.db[0].id
   db_proxy_name          = aws_db_proxy.rds_proxy[0].name
   target_group_name      = aws_db_proxy_default_target_group.default.name
-
+  lifecycle {
+    ignore_changes = [ db_instance_identifier ]
+    replace_triggered_by = [ data.aws_rds_cluster.db ]
+  }
   depends_on = [ aws_db_proxy.rds_proxy ]
 }
 
@@ -199,7 +205,7 @@ resource "aws_iam_policy" "rds_proxy_iam" {
                 "secretsmanager:ListSecretVersionIds"
             ],
             "Resource": [
-                "${data.aws_secretsmanager_secret_version.database_credentials.arn}"
+                "arn:aws:secretsmanager:${data.aws_region.current.name}.${data.aws_caller_identity.current.account_id}:secret:${var.aws_db_proxy_secret_name}*"
             ]
         },
         {
@@ -215,6 +221,9 @@ resource "aws_iam_policy" "rds_proxy_iam" {
     ]
 }
 POLICY
+  lifecycle {
+    ignore_changes = [ policy ]
+  }
 }
 
 resource "aws_iam_role_policy_attachment" "rds_policy" {

operations/deployment/terraform/modules/aws/rds/aws_rds.tf

@@ -53,7 +53,7 @@ resource "aws_db_subnet_group" "selected" {
 }
 
 resource "aws_db_instance" "default" {
-  identifier                       = var.aws_rds_db_name != null ? var.aws_rds_db_name : var.aws_resource_identifier
+  identifier                       = var.aws_rds_db_identifier
   engine                           = var.aws_rds_db_engine
   engine_version                   = var.aws_rds_db_engine_version
   db_subnet_group_name             = aws_db_subnet_group.selected.name
@@ -98,13 +98,19 @@ resource "aws_secretsmanager_secret_version" "database_credentials_sm_secret_ver
   secret_string = jsonencode({
    username          = sensitive(aws_db_instance.default.username)
    password          = sensitive(aws_db_instance.default.password)
-   DB_ENGINE         = sensitive(aws_db_instance.default.engine)
-   DB_ENGINE_VERSION = sensitive(aws_db_instance.default.engine_version)
+   host              = sensitive(aws_db_instance.default.address)
+   port              = sensitive(aws_db_instance.default.port)
+   database          = sensitive(aws_db_instance.default.db_name)
+   engine            = sensitive(aws_db_instance.default.engine)
+   engine_version    = sensitive(aws_db_instance.default.engine_version)
    DB_USER           = sensitive(aws_db_instance.default.username)
+   DB_USERNAME       = sensitive(aws_db_instance.default.username)
    DB_PASSWORD       = sensitive(aws_db_instance.default.password)
-   DB_NAME           = sensitive(aws_db_instance.default.db_name)
-   DB_PORT           = sensitive(aws_db_instance.default.port)
    DB_HOST           = sensitive(aws_db_instance.default.address)
+   DB_PORT           = sensitive(aws_db_instance.default.port)
+   DB_NAME           = sensitive(aws_db_instance.default.db_name)
+   DB_ENGINE         = sensitive(aws_db_instance.default.engine)
+   DB_ENGINE_VERSION = sensitive(aws_db_instance.default.engine_version)
   })
 }
 

operations/deployment/terraform/modules/aws/rds/aws_rds_vars.tf

@@ -1,3 +1,4 @@
+variable "aws_rds_db_identifier" {}
 variable "aws_rds_db_name" {}
 variable "aws_rds_db_user" {}
 variable "aws_rds_db_engine" {}

operations/deployment/terraform/modules/aws/secretmanager_get/aws_secretmanager_get.tf

@@ -1,15 +1,20 @@
 # This file will create a key=value file with an AWS Secret stored in AWS Secret Manager
 # With a JSON style of "{"key1":"value1","key2":"value2"}"
-data "aws_secretsmanager_secret_version" "env_secret" {
-  secret_id = var.env_aws_secret
+data "aws_secretsmanager_secret_version" "secret_list" {
+  count = length(local.env_aws_secret)
+  secret_id = local.env_aws_secret[count.index] 
 }
 
 resource "local_file" "tf-secretdotenv" {
   filename = format("%s/%s", abspath(path.root), "aws.env")
-  content  = "${local.s3_secret_string}\n"
+  content  =  join("\n",local.s3_secret_string)
 }
 
 locals {
-  s3_secret_raw    = nonsensitive(jsondecode(data.aws_secretsmanager_secret_version.env_secret.secret_string))
-  s3_secret_string = join("\n", [for k, v in local.s3_secret_raw : "${k}=\"${v}\""])
+  env_aws_secret = [for n in split(",", var.env_aws_secret) : (n)] 
+  all_secret_contents = {
+    for secret_name, secret_data in data.aws_secretsmanager_secret_version.secret_list : secret_name => jsondecode(secret_data.secret_string)
+  }
+  merged_secrets = merge(values(local.all_secret_contents)...)
+  s3_secret_string = [for key, value in local.merged_secrets : "${key}=${value}"]
 }