Terraform splitting (#26)

* File location moving

* Fixing terraform.tfvars

* BIG changes in generate_bitops_config

* Moving more files out of generators

* DB Snap lookup fix

* Big bump to DB things

* Bumping engine_version for postgres to 11.17

* Dupe var fix

* Moving EFS

* Moved RDS module

* Fixes in vars references

* Eliminating deps from R53 to Certs

* Moved ELB

* cert for elb fix

* Fixing Aurora module calling index

* Fixes and moving EKS

* Dupe removal

* Fixing some cert issue in ELB - adding outputs

* Tyiding things up - testing efs sg issue fix

* env_merger debugging

* EFS Access only from VPC to NFS port

* Tidy up

* Huge movement - EC2 to module, generator completely removed

* Code cleanup

---------

Co-authored-by: Max Cascone <[email protected]>

Author: LeoDiazL

.github/workflows/unit_tests.yaml

@@ -0,0 +1,21 @@
+name: Unit Tests
+
+on:
+  - push
+  - workflow_dispatch
+  
+permissions:
+  contents: read
+
+jobs:
+  Unit-Tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup BATS
+      uses: mig4/setup-bats@v1
+
+    - name: checkout
+      uses: actions/checkout@v2
+
+    - name: unit tests
+      run: bats $GITHUB_WORKSPACE/operations/_scripts/deploy/tests/.

.gitignore

@@ -0,0 +1,20 @@
+local.sh
+*.env
+
+# Terraform
+*.terraform.lock*
+.terraform*
+
+# BitOps
+bitops.config.yaml
+
+# BitOps GHAs
+provider.tf
+terraform.tfvars
+
+terraform-*
+operations/deployment/terraform/inventory.yaml
+
+*.pem
+
+.vscode/

README.md

@@ -197,13 +197,17 @@ The following inputs can be used as `step.with` keys
 |------------------|---------|------------------------------------|
 | `aws_postgres_enable` | Boolean | Set to `true` to enable a postgres database. |
 | `aws_postgres_engine` | String |  Which Database engine to use. Default is `aurora-postgresql`.|
-| `aws_postgres_engine_version` | String |  Specify Postgres version.  More information [here](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraPostgreSQL.Updates.20180305.html). Default is `11.13`. |
+| `aws_postgres_engine_version` | String |  Specify Postgres version.  More information [here](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraPostgreSQL.Updates.20180305.html). Default is `11.17`. |
 | `aws_postgres_database_group_family` | String | Specify aws database group family. Default is `aurora-postgresql11`. See [this](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/rds/create-db-parameter-group.html).|
 | `aws_postgres_instance_class` | String | Define the size of the instances in the DB cluster. Default is `db.t3.medium`. | 
 | `aws_postgres_security_group_name` | String | The name of the Postgres security group. Defaults to `SG for ${aws_resource_identifier} - PG`. |
 | `aws_postgres_subnets` | String | Specify which subnets to use as a list of strings.  Example: `i-1234,i-5678,i-9101`. |
-| `aws_postgres_database_name` | String | Specify a database name. Will be created if it does not exist. Default is `root`. |
+| `aws_postgres_cluster_name` | String | Specify a cluster name. Will be created if it does not exist. Defaults to `aws_resource_identifier`. |
+| `aws_postgres_database_name` | String | Specify a database name. Will be created if it does not exist. Defaults to `aws_resource_identifier`. |
 | `aws_postgres_database_port` | String | Specify a listening port for the database. Default is `5432`.|
+| `aws_postgres_restore_snapshot` | String | Restore a snapshot to the DB. Should be set only once. Changes in this value will destroy and recreate the database completely. | 
+| `aws_postgres_snapshot_name` | String | Specify a database name. Will be created if it does not exist. Won't overwrite. |
+| `aws_postgres_snapshot_overwrite` | Boolean | Set to true to overwrite the snapshot. |
 | `aws_postgres_database_protection` | Boolean | Protects the database from deletion. Default is `false`.|
 | `aws_postgres_database_final_snapshot` | Boolean | Creates a snapshot before deletion. If a string is passed, it will be used as snapsthot name. Defaults to `false`.|
 <hr/>

action.yaml

@@ -274,12 +274,24 @@ inputs:
   aws_postgres_subnets:
     description: 'Specify which subnets to use as a list of strings.  Example: `i-1234,i-5678,i-9101`'
     required: false
+  aws_postgres_cluster_name:
+    description: 'Specify a cluster name. Will be created if it does not exist'
+    required: false
   aws_postgres_database_name:
     description: 'Specify a database name. Will be created if it does not exist'
     required: false
   aws_postgres_database_port:
     description: 'Postgres database port'
     required: false
+  aws_postgres_restore_snapshot:
+    description: 'Restore a snapshot to the DB. Should be used only once. Changes in this value will destroy and recreate the database completely.'
+    required: false
+  aws_postgres_snapshot_name:
+    description: 'Takes a snapshot of the cluster using that name. If none definded, no snapshot will be made. If snap already exists, no new one will be created.'
+    required: false
+  aws_postgres_snapshot_overwrite:
+    description: 'If the snapshot name is the same as an existing one, will destroy and create a new one.'
+    required: false
   aws_postgres_database_protection:
     description: 'Protects the database from deletion.'
     required: false
@@ -492,8 +504,12 @@ runs:
         AWS_POSTGRES_INSTANCE_CLASS: ${{ inputs.aws_postgres_instance_class }}
         AWS_POSTGRES_SECURITY_GROUP_NAME: ${{ inputs.aws_postgres_security_group_name }}
         AWS_POSTGRES_SUBNETS: ${{ inputs.aws_postgres_subnets }}
+        AWS_POSTGRES_CLUSTER_NAME: ${{ inputs.aws_postgres_cluster_name }}
         AWS_POSTGRES_DATABASE_NAME: ${{ inputs.aws_postgres_database_name }}
         AWS_POSTGRES_DATABASE_PORT: ${{ inputs.aws_postgres_database_port}}
+        AWS_POSTGRES_RESTORE_SNAPSHOT: ${{ inputs.aws_postgres_restore_snapshot }}
+        AWS_POSTGRES_SNAPSHOT_NAME: ${{ inputs.aws_postgres_snapshot_name }}
+        AWS_POSTGRES_SNAPSHOT_OVERWRITE: ${{ inputs.aws_postgres_snapshot_overwrite }}
         AWS_POSTGRES_DATABASE_PROTECTION: ${{ inputs.aws_postgres_database_protection }}
         AWS_POSTGRES_DATABASE_FINAL_SNAPSHOT: ${{ inputs.aws_postgres_database_final_snapshot }}
 
@@ -535,53 +551,20 @@ runs:
         $GITHUB_ACTION_PATH/operations/_scripts/deploy/export_vars.sh
         echo "Finished executions - Now to print results"
 
-    # output results to GitHub
-    - if: ${{ success() && steps.deploy.outputs.vm_url != '' }}
-      name: Print result created
-      shell: bash
-      run: |
-        echo "## VM Created! :rocket:" >> $GITHUB_STEP_SUMMARY
-        echo " ${{ steps.deploy.outputs.vm_url }}" >> $GITHUB_STEP_SUMMARY
-    - if: ${{ success() && steps.deploy.outputs.vm_url == '' && inputs.bitops_code_only == 'true' && inputs.bitops_code_store == 'true' }}
-      name: Print code generated and archived
+    # always output results to GitHub Summary UI
+    - name: Generate Summary Output
+      if: ${{ always() }}
       shell: bash
-      run: |
-        echo "## BitOps Code generated. :tada: " >> $GITHUB_STEP_SUMMARY
-        echo "Download the code artifact. Will be there for 5 days." >> $GITHUB_STEP_SUMMARY
-        echo "Keep in mind that for creation, EFS should be created before EC2."
-        echo "While destroying, EC2 should be destroyed before EFS. (Due to resources being in use)."
-        echo "You can change that in the bitops.config.yaml file, or regenerate the code with destroy set."
-    - if: ${{ success() && steps.deploy.outputs.vm_url == '' && inputs.bitops_code_only == 'true' && inputs.bitops_code_store != 'true' }}
-      name: Print code built not archived
-      shell: bash
-      run: |
-        echo "## BitOps Code generated. :tada: " >> $GITHUB_STEP_SUMMARY
-    - if: ${{ success() && steps.deploy.outputs.vm_url == '' && inputs.tf_stack_destroy != 'true' && inputs.bitops_code_only != 'true' }}
-      name: Print result deploy finished but no URL.
-      shell: bash
-      run: |
-        echo "## Deploy finished! But no URL found. :thinking: " >> $GITHUB_STEP_SUMMARY
-        echo "If expecting an URL, please check the logs for possible  errors." >> $GITHUB_STEP_SUMMARY
-        echo "If you consider this is a bug in the Github Action, please submit an issue to our repo." >> $GITHUB_STEP_SUMMARY
-    - if: ${{ success() && steps.deploy.outputs.vm_url == '' && inputs.tf_stack_destroy == 'true' && inputs.tf_state_bucket_destroy != 'true' }}
-      name: Print result destroyed no bucket
-      shell: bash
-      run: |
-        echo "## VM Destroyed! :boom:" >> $GITHUB_STEP_SUMMARY
-        echo "Infrastructure should be gone now!" >> $GITHUB_STEP_SUMMARY
-    - if: ${{ success() && steps.deploy.outputs.vm_url == '' && inputs.tf_stack_destroy == 'true' && inputs.tf_state_bucket_destroy == 'true' }}
-      name: Print result destroyed with bucket
-      shell: bash
-      run: |
-        echo "## VM Destroyed! :boom:" >> $GITHUB_STEP_SUMMARY
-        echo "Buckets and infrastructure should be gone now!" >> $GITHUB_STEP_SUMMARY
-    - if: ${{ failure() }} 
-      name: Print error result
-      shell: bash
-      run: |
-        echo "## Workflow failed to run :fire:" >> $GITHUB_STEP_SUMMARY
-        echo "Please check the logs for possible errors." >> $GITHUB_STEP_SUMMARY
-        echo "If you consider this is a bug in the Github Action, please submit an issue to our repo." >> $GITHUB_STEP_SUMMARY
+      env: 
+        SUCCESS: ${{ job.status }} # success, failure, cancelled
+        URL_OUTPUT: ${{ steps.deploy.outputs.vm_url }}
+        BITOPS_CODE_ONLY: ${{ inputs.bitops_code_only }}
+        BITOPS_CODE_STORE: ${{ inputs.bitops_code_store }}
+        TF_STACK_DESTROY: ${{ inputs.tf_stack_destroy }}
+        TF_STATE_BUCKET_DESTROY: ${{ inputs.tf_state_bucket_destroy }}
+      run: $GITHUB_ACTION_PATH/operations/_scripts/deploy/summary.sh
+
+    # upload generated artifacts to GitHub if enabled
     - if: ${{ inputs.bitops_code_store == 'true' }}
       name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -590,4 +573,4 @@ runs:
         retention-days: 5
         path: |
           ${{ github.action_path }}/operations/generated_code
-          !${{ github.action_path }}/operations/generated_code/**/ghs.env
+          !${{ github.action_path }}/operations/generated_code/**/ghs.env

operations/_scripts/deploy/deploy.sh

@@ -87,6 +87,8 @@ if [ -s "$GITHUB_WORKSPACE/$ENV_REPO" ] && [ -n "$ENV_REPO" ]; then
   cp "$GITHUB_WORKSPACE/$ENV_REPO" "${GITHUB_ACTION_PATH}/operations/deployment/env-files/repo.env"
 fi
 
+cp -r "$GITHUB_ACTION_PATH/operations" /opt/bitops_deployment/generated_code
+
 if [[ $(alpha_only "$BITOPS_SKIP_RUN") == true ]]; then
   echo "BitOps skip run is set to true. Reached end of the line."
   exit 0
@@ -98,7 +100,6 @@ for i in $(env | grep BITOPS_); do
   BITOPS_EXTRA_ENV_VARS="${BITOPS_EXTRA_ENV_VARS} -e ${i}"
 done
 
-echo "BITOPS_EXTRA_ENV_VARS: $BITOPS_EXTRA_ENV_VARS"
 
 echo "::group::BitOps Excecution"  
 echo "Running BitOps for env: $BITOPS_ENVIRONMENT"

operations/_scripts/deploy/summary.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# shellcheck disable=SC2086
+
+### coming into this we have env vars:
+# SUCCESS=${{ job.status }} # success, cancelled, failure
+# URL_OUTPUT=${{ steps.deploy.outputs.vm_url }}
+# BITOPS_CODE_ONLY
+# BITOPS_CODE_STORE
+# TF_STACK_DESTROY
+# TF_STATE_BUCKET_DESTROY
+
+# Create an error code mechanism so we don't have to check the actual static text,
+# just which case we fell into
+
+# 0 - success
+# 1 - failure
+# 2 - failure, no URL # invalid case
+# 3 - failure, no URL, no code generated # invalid case
+# 4 - success, no URL
+# 5 - success, code generated, not archived
+# 6 - success, code generated, archived
+# 7 - success, code generated, archived, but no URL found # invalid case
+# 8 - success, destroy buckets and infrastructure
+# 9 - success, destroy infrastructure
+# 10 - cancelled
+
+SUMMARY_CODE=0
+
+if [[ $SUCCESS == 'success' ]]; then
+  if [[ $URL_OUTPUT != '' ]]; then
+    result_string="## Deploy Complete! :rocket:
+    $URL_OUTPUT"
+
+  elif [[ $BITOPS_CODE_ONLY == 'true' ]]; then
+    if [[ $BITOPS_CODE_STORE == 'true' ]]; then
+      SUMMARY_CODE=6
+      result_string="## BitOps Code generated. :tada: 
+      Download the code artifact. Will be there for 5 days.
+      Keep in mind that for creation, EFS should be created before EC2.
+      While destroying, EC2 should be destroyed before EFS. (Due to resources being in use).
+      You can change that in the bitops.config.yaml file, or regenerate the code with destroy set."
+    else
+      SUMMARY_CODE=5
+      result_string="## BitOps Code generated. :tada:"
+    fi
+
+  elif [[ $TF_STACK_DESTROY == 'true' ]]; then
+    if [[ $TF_STATE_BUCKET_DESTROY != 'true' ]]; then
+      SUMMARY_CODE=9
+      result_string="## VM Destroyed! :boom:
+      Infrastructure should be gone now!"
+    else
+      SUMMARY_CODE=8
+      result_string="## VM Destroyed! :boom:
+      Buckets and infrastructure should be gone now!"
+    fi
+
+  elif [[ $TF_STACK_DESTROY != 'true' && $BITOPS_CODE_ONLY != 'true' ]]; then
+    SUMMARY_CODE=4
+    result_string="## Deploy finished! But no URL found. :thinking:
+    If expecting a URL, please check the logs for possible errors.
+    If you consider this is a bug in the Github Action, please submit an issue to our repo."
+  fi
+elif [[ $SUCCESS == 'cancelled' ]]; then
+  SUMMARY_CODE=10
+  result_string="## Workflow cancelled :warning:"
+
+else
+  SUMMARY_CODE=1
+  result_string="## Workflow failed to run :fire:
+  Please check the logs for possible errors.
+  If you consider this is a bug in the Github Action, please submit an issue to our repo."
+fi
+
+echo "$result_string" >> $GITHUB_STEP_SUMMARY
+echo "SUMMARY_CODE=$SUMMARY_CODE" >> $GITHUB_OUTPUT

operations/_scripts/deploy/tests/summary_tests.bats

@@ -0,0 +1,97 @@
+#!/usr/bin/env bats
+
+# see here for summary_codes
+# this runs once before all tests
+setup_file() {
+  export file_under_test=$GITHUB_WORKSPACE/operations/_scripts/deploy/summary.sh
+  export GITHUB_STEP_SUMMARY=$(mktemp ./test.XXX)
+  export GITHUB_OUTPUT=$(mktemp ./test.XXX)
+}
+
+# this runs once after all tests
+teardown_file() {
+  rm -f $GITHUB_STEP_SUMMARY $GITHUB_OUTPUT
+}
+
+function runTest() {
+  # Run the script
+  source $file_under_test
+
+  # take the passed-in expected result code
+  expected_result=$1
+
+  # Compare the results
+  # SUMMARY_CODE is set in summary script as an output
+  source $GITHUB_OUTPUT
+  [[ "$SUMMARY_CODE" = "$expected_result" ]]
+}
+
+@test "SUCCESS is true, URL_OUTPUT is not empty" {
+  # Set the environment variables
+  export SUCCESS='success'
+  export URL_OUTPUT='example.com'
+
+  # Run the test and pass in the expected result code
+  runTest 0
+}
+
+@test "SUCCESS is true, URL_OUTPUT is empty, BITOPS_CODE_ONLY is true, BITOPS_CODE_STORE is true" {
+  export SUCCESS='success'
+  export URL_OUTPUT=''
+  export BITOPS_CODE_ONLY='true'
+  export BITOPS_CODE_STORE='true'
+
+  runTest 6
+}
+
+@test "SUCCESS is true, URL_OUTPUT is empty, BITOPS_CODE_ONLY is true, BITOPS_CODE_STORE is false" {
+  export SUCCESS='success'
+  export URL_OUTPUT=''
+  export BITOPS_CODE_ONLY='true'
+  export BITOPS_CODE_STORE='false'
+
+  runTest 5
+}
+
+@test "SUCCESS is true, URL_OUTPUT is empty, TF_STACK_DESTROY is true, TF_STATE_BUCKET_DESTROY is false" {
+  export SUCCESS='success'
+  export URL_OUTPUT=''
+  export BITOPS_CODE_ONLY='false'
+  export TF_STACK_DESTROY='true'
+  export TF_STATE_BUCKET_DESTROY='false'
+
+  runTest 9
+}
+
+@test "SUCCESS is true, URL_OUTPUT is empty, TF_STACK_DESTROY is true, TF_STATE_BUCKET_DESTROY is true" {
+  export SUCCESS='success'
+  export URL_OUTPUT=''
+  export BITOPS_CODE_ONLY='false'
+  export TF_STACK_DESTROY='true'
+  export TF_STATE_BUCKET_DESTROY='true'
+
+  runTest 8
+}
+
+@test "SUCCESS is true, URL_OUTPUT is empty, BITOPS_CODE_ONLY is false, TF_STACK_DESTROY is false" {
+  export SUCCESS='success'
+  export URL_OUTPUT=''
+  export BITOPS_CODE_ONLY='false'
+  export TF_STACK_DESTROY='false'
+  export BITOPS_CODE_STORE='false'
+
+  runTest 4
+}
+
+@test "SUCCESS is false" {
+  export SUCCESS='false'
+
+  runTest 1
+
+}
+
+@test "SUCCESS is 'cancelled'" {
+  export SUCCESS='cancelled'
+
+  runTest 10
+}