Make api gateway optional and adjust api_base_auth var naming (#40)

* Make api gateway optional and adjust api_base_auth var naming

Co-authored-by: Hugues Alary <[email protected]>

* Format api-gateway.tf file

* Use metadata_service_enable_api_basic_auth var name from root of module

---------

Co-authored-by: Hugues Alary <[email protected]>

Author: Erin-Boehmer

README.md

@@ -96,7 +96,6 @@ You can find a more complete example that uses this module but also includes set
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_access_list_cidr_blocks"></a> [access\_list\_cidr\_blocks](#input\_access\_list\_cidr\_blocks) | List of CIDRs we want to grant access to our Metaflow Metadata Service. Usually this is our VPN's CIDR blocks. | `list(string)` | `[]` | no |
-| <a name="input_api_basic_auth"></a> [api\_basic\_auth](#input\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
 | <a name="input_batch_type"></a> [batch\_type](#input\_batch\_type) | AWS Batch Compute Type ('ec2', 'fargate') | `string` | `"ec2"` | no |
 | <a name="input_compute_environment_desired_vcpus"></a> [compute\_environment\_desired\_vcpus](#input\_compute\_environment\_desired\_vcpus) | Desired Starting VCPUs for Batch Compute Environment [0-16] for EC2 Batch Compute Environment (ignored for Fargate) | `number` | `8` | no |
 | <a name="input_compute_environment_egress_cidr_blocks"></a> [compute\_environment\_egress\_cidr\_blocks](#input\_compute\_environment\_egress\_cidr\_blocks) | CIDR blocks to which egress is allowed from the Batch Compute environment's security group | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
@@ -112,6 +111,8 @@ You can find a more complete example that uses this module but also includes set
 | <a name="input_launch_template_http_put_response_hop_limit"></a> [launch\_template\_http\_put\_response\_hop\_limit](#input\_launch\_template\_http\_put\_response\_hop\_limit) | The desired HTTP PUT response hop limit for instance metadata requests. Can be an integer from 1 to 64 | `number` | `2` | no |
 | <a name="input_launch_template_http_tokens"></a> [launch\_template\_http\_tokens](#input\_launch\_template\_http\_tokens) | Whether or not the metadata service requires session tokens, also referred to as Instance Metadata Service Version 2 (IMDSv2). Can be 'optional' or 'required' | `string` | `"optional"` | no |
 | <a name="input_metadata_service_container_image"></a> [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no |
+| <a name="input_metadata_service_enable_api_basic_auth"></a> [metadata\_service\_enable\_api\_basic\_auth](#input\_metadata\_service\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
+| <a name="input_metadata_service_enable_api_gateway"></a> [metadata\_service\_enable\_api\_gateway](#input\_metadata\_service\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no |
 | <a name="input_resource_prefix"></a> [resource\_prefix](#input\_resource\_prefix) | string prefix for all resources | `string` | `"metaflow"` | no |
 | <a name="input_resource_suffix"></a> [resource\_suffix](#input\_resource\_suffix) | string suffix for all resources | `string` | `""` | no |
 | <a name="input_subnet1_id"></a> [subnet1\_id](#input\_subnet1\_id) | First subnet used for availability zone redundancy | `string` | n/a | yes |

examples/eks_airflow/metaflow.tf

@@ -48,11 +48,12 @@ module "metaflow-metadata-service" {
   resource_suffix = local.resource_suffix
 
   access_list_cidr_blocks          = []
-  api_basic_auth                   = true
   database_name                    = module.metaflow-datastore.database_name
   database_password                = module.metaflow-datastore.database_password
   database_username                = module.metaflow-datastore.database_username
   datastore_s3_bucket_kms_key_arn  = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
+  enable_api_basic_auth            = true
+  enable_api_gateway               = true
   fargate_execution_role_arn       = aws_iam_role.ecs_execution_role.arn
   metaflow_vpc_id                  = module.vpc.vpc_id
   metadata_service_container_image = module.metaflow-common.default_metadata_service_container_image

main.tf

@@ -19,11 +19,12 @@ module "metaflow-metadata-service" {
   resource_suffix = local.resource_suffix
 
   access_list_cidr_blocks          = var.access_list_cidr_blocks
-  api_basic_auth                   = var.api_basic_auth
   database_name                    = module.metaflow-datastore.database_name
   database_password                = module.metaflow-datastore.database_password
   database_username                = module.metaflow-datastore.database_username
   datastore_s3_bucket_kms_key_arn  = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
+  enable_api_basic_auth            = var.metadata_service_enable_api_basic_auth
+  enable_api_gateway               = var.metadata_service_enable_api_gateway
   fargate_execution_role_arn       = module.metaflow-computation.ecs_execution_role_arn
   iam_partition                    = var.iam_partition
   metadata_service_container_image = local.metadata_service_container_image

modules/metadata-service/README.md

@@ -16,11 +16,12 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_access_list_cidr_blocks"></a> [access\_list\_cidr\_blocks](#input\_access\_list\_cidr\_blocks) | List of CIDRs we want to grant access to our Metaflow Metadata Service. Usually this is our VPN's CIDR blocks. | `list(string)` | n/a | yes |
-| <a name="input_api_basic_auth"></a> [api\_basic\_auth](#input\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
 | <a name="input_database_name"></a> [database\_name](#input\_database\_name) | The database name | `string` | `"metaflow"` | no |
 | <a name="input_database_password"></a> [database\_password](#input\_database\_password) | The database password | `string` | n/a | yes |
 | <a name="input_database_username"></a> [database\_username](#input\_database\_username) | The database username | `string` | n/a | yes |
 | <a name="input_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#input\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket | `string` | n/a | yes |
+| <a name="input_enable_api_basic_auth"></a> [enable\_api\_basic\_auth](#input\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
+| <a name="input_enable_api_gateway"></a> [enable\_api\_gateway](#input\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no |
 | <a name="input_fargate_execution_role_arn"></a> [fargate\_execution\_role\_arn](#input\_fargate\_execution\_role\_arn) | The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution\_role for our Fargate instance | `string` | n/a | yes |
 | <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
 | <a name="input_is_gov"></a> [is\_gov](#input\_is\_gov) | Set to true if IAM partition is 'aws-us-gov' | `bool` | `false` | no |

modules/metadata-service/api-gateway.tf

@@ -1,6 +1,6 @@
 resource "aws_api_gateway_rest_api_policy" "this" {
-  count       = length(var.access_list_cidr_blocks) > 0 ? 1 : 0
-  rest_api_id = aws_api_gateway_rest_api.this.id
+  count       = var.enable_api_gateway && length(var.access_list_cidr_blocks) > 0 ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
   policy      = <<EOF
   {
     "Version": "2012-10-17",
@@ -9,13 +9,13 @@ resource "aws_api_gateway_rest_api_policy" "this" {
             "Effect": "Allow",
             "Principal": "*",
             "Action": "execute-api:Invoke",
-            "Resource": "arn:${var.iam_partition}:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.this.id}/*/*/*"
+            "Resource": "arn:${var.iam_partition}:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.this[0].id}/*/*/*"
         },
         {
             "Effect": "Deny",
             "Principal": "*",
             "Action": "execute-api:Invoke",
-            "Resource": "arn:${var.iam_partition}:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.this.id}/*/*/*",
+            "Resource": "arn:${var.iam_partition}:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.this[0].id}/*/*/*",
             "Condition": {
                 "NotIpAddress": {
                     "aws:SourceIp": ${jsonencode(var.access_list_cidr_blocks)}
@@ -28,6 +28,7 @@ resource "aws_api_gateway_rest_api_policy" "this" {
 }
 
 resource "aws_api_gateway_rest_api" "this" {
+  count       = var.enable_api_gateway ? 1 : 0
   name        = "${var.resource_prefix}api${var.resource_suffix}"
   description = "Allows access to the Metadata service RDS instance"
 
@@ -39,56 +40,63 @@ resource "aws_api_gateway_rest_api" "this" {
 }
 
 resource "aws_api_gateway_resource" "this" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  parent_id   = aws_api_gateway_rest_api.this.root_resource_id
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  parent_id   = aws_api_gateway_rest_api.this[0].root_resource_id
   path_part   = "{proxy+}"
 }
 
 resource "aws_api_gateway_resource" "db" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  parent_id   = aws_api_gateway_rest_api.this.root_resource_id
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  parent_id   = aws_api_gateway_rest_api.this[0].root_resource_id
   path_part   = "db_schema_status"
 }
 
 resource "aws_api_gateway_vpc_link" "this" {
+  count       = var.enable_api_gateway ? 1 : 0
   name        = "${var.resource_prefix}vpclink${var.resource_suffix}"
   target_arns = [aws_lb.this.arn]
 
   tags = var.standard_tags
 }
 
 resource "aws_api_gateway_method" "this" {
+  count            = var.enable_api_gateway ? 1 : 0
   http_method      = "ANY"
-  resource_id      = aws_api_gateway_resource.this.id
-  rest_api_id      = aws_api_gateway_rest_api.this.id
+  resource_id      = aws_api_gateway_resource.this[0].id
+  rest_api_id      = aws_api_gateway_rest_api.this[0].id
   authorization    = "NONE"
-  api_key_required = var.api_basic_auth
+  api_key_required = var.enable_api_basic_auth
 
   request_parameters = {
     "method.request.path.proxy" = true
   }
 }
 
 resource "aws_api_gateway_method" "db" {
+  count            = var.enable_api_gateway ? 1 : 0
   http_method      = "GET"
-  resource_id      = aws_api_gateway_resource.db.id
-  rest_api_id      = aws_api_gateway_rest_api.this.id
+  resource_id      = aws_api_gateway_resource.db[0].id
+  rest_api_id      = aws_api_gateway_rest_api.this[0].id
   authorization    = "NONE"
-  api_key_required = var.api_basic_auth
+  api_key_required = var.enable_api_basic_auth
 }
 
 resource "aws_api_gateway_integration_response" "this" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  resource_id = aws_api_gateway_resource.this.id
-  http_method = aws_api_gateway_method.this.http_method
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  resource_id = aws_api_gateway_resource.this[0].id
+  http_method = aws_api_gateway_method.this[0].http_method
   status_code = 200
-  depends_on  = [aws_api_gateway_integration.this]
+  depends_on  = [aws_api_gateway_integration.this[0]]
 }
 
 resource "aws_api_gateway_integration" "this" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  resource_id = aws_api_gateway_resource.this.id
-  http_method = aws_api_gateway_method.this.http_method
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  resource_id = aws_api_gateway_resource.this[0].id
+  http_method = aws_api_gateway_method.this[0].http_method
 
   request_parameters = {
     "integration.request.path.proxy" = "method.request.path.proxy"
@@ -99,46 +107,50 @@ resource "aws_api_gateway_integration" "this" {
   integration_http_method = "ANY"
   passthrough_behavior    = "WHEN_NO_MATCH"
   connection_type         = "VPC_LINK"
-  connection_id           = aws_api_gateway_vpc_link.this.id
+  connection_id           = aws_api_gateway_vpc_link.this[0].id
 }
 
 resource "aws_api_gateway_integration" "db" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  resource_id = aws_api_gateway_resource.db.id
-  http_method = aws_api_gateway_method.db.http_method
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  resource_id = aws_api_gateway_resource.db[0].id
+  http_method = aws_api_gateway_method.db[0].http_method
 
 
   type                    = "HTTP_PROXY"
   uri                     = "http://${aws_lb.this.dns_name}:8082/db_schema_status"
   integration_http_method = "GET"
   passthrough_behavior    = "WHEN_NO_MATCH"
   connection_type         = "VPC_LINK"
-  connection_id           = aws_api_gateway_vpc_link.this.id
+  connection_id           = aws_api_gateway_vpc_link.this[0].id
 }
 
 resource "aws_api_gateway_method_response" "this" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  resource_id = aws_api_gateway_resource.this.id
-  http_method = aws_api_gateway_method.this.http_method
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  resource_id = aws_api_gateway_resource.this[0].id
+  http_method = aws_api_gateway_method.this[0].http_method
   status_code = "200"
-  depends_on  = [aws_api_gateway_integration.this]
+  depends_on  = [aws_api_gateway_integration.this[0]]
 }
 
 resource "aws_api_gateway_method_response" "db" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
-  resource_id = aws_api_gateway_resource.db.id
-  http_method = aws_api_gateway_method.db.http_method
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
+  resource_id = aws_api_gateway_resource.db[0].id
+  http_method = aws_api_gateway_method.db[0].http_method
   status_code = "200"
-  depends_on  = [aws_api_gateway_integration.db]
+  depends_on  = [aws_api_gateway_integration.db[0]]
 }
 
 resource "aws_api_gateway_deployment" "this" {
-  rest_api_id = aws_api_gateway_rest_api.this.id
+  count       = var.enable_api_gateway ? 1 : 0
+  rest_api_id = aws_api_gateway_rest_api.this[0].id
 
   # explicit depends_on required to ensure module stands up on first `apply`
   # otherwise a second followup `apply` would be required
   # can read more here: https://stackoverflow.com/a/42783769
-  depends_on = [aws_api_gateway_method.this, aws_api_gateway_integration.this]
+  depends_on = [aws_api_gateway_method.this[0], aws_api_gateway_integration.this[0]]
 
   # ensures properly ordered re-deployments occur
   lifecycle {
@@ -147,34 +159,35 @@ resource "aws_api_gateway_deployment" "this" {
 }
 
 resource "aws_api_gateway_stage" "this" {
-  deployment_id = aws_api_gateway_deployment.this.id
-  rest_api_id   = aws_api_gateway_rest_api.this.id
+  count         = var.enable_api_gateway ? 1 : 0
+  deployment_id = aws_api_gateway_deployment.this[0].id
+  rest_api_id   = aws_api_gateway_rest_api.this[0].id
   stage_name    = local.api_gateway_stage_name
 
   tags = var.standard_tags
 }
 
 resource "aws_api_gateway_api_key" "this" {
-  count = var.api_basic_auth ? 1 : 0
+  count = var.enable_api_gateway && var.enable_api_basic_auth ? 1 : 0
   name  = local.api_gateway_key_name
 
   tags = var.standard_tags
 }
 
 resource "aws_api_gateway_usage_plan" "this" {
-  count = var.api_basic_auth ? 1 : 0
+  count = var.enable_api_gateway && var.enable_api_basic_auth ? 1 : 0
   name  = local.api_gateway_usage_plan_name
 
   api_stages {
-    api_id = aws_api_gateway_rest_api.this.id
-    stage  = aws_api_gateway_stage.this.stage_name
+    api_id = aws_api_gateway_rest_api.this[0].id
+    stage  = aws_api_gateway_stage.this[0].stage_name
   }
 
   tags = var.standard_tags
 }
 
 resource "aws_api_gateway_usage_plan_key" "this" {
-  count         = var.api_basic_auth ? 1 : 0
+  count         = var.enable_api_gateway && var.enable_api_basic_auth ? 1 : 0
   key_id        = aws_api_gateway_api_key.this[0].id
   key_type      = "API_KEY"
   usage_plan_id = aws_api_gateway_usage_plan.this[0].id

modules/metadata-service/outputs.tf

@@ -4,12 +4,12 @@ output "METAFLOW_SERVICE_INTERNAL_URL" {
 }
 
 output "METAFLOW_SERVICE_URL" {
-  value       = "https://${aws_api_gateway_rest_api.this.id}.execute-api.${data.aws_region.current.name}.amazonaws.com/api/"
+  value       = var.enable_api_gateway ? "https://${aws_api_gateway_rest_api.this[0].id}.execute-api.${data.aws_region.current.name}.amazonaws.com/api/" : ""
   description = "URL for Metadata Service (Open to Public Access)"
 }
 
 output "api_gateway_rest_api_id" {
-  value       = aws_api_gateway_rest_api.this.id
+  value       = var.enable_api_gateway ? aws_api_gateway_rest_api.this[0].id : ""
   description = "The ID of the API Gateway REST API we'll use to accept MetaData service requests to forward to the Fargate API instance"
 }
 

modules/metadata-service/variables.tf

@@ -3,11 +3,6 @@ variable "access_list_cidr_blocks" {
   description = "List of CIDRs we want to grant access to our Metaflow Metadata Service. Usually this is our VPN's CIDR blocks."
 }
 
-variable "api_basic_auth" {
-  type        = bool
-  default     = true
-  description = "Enable basic auth for API Gateway? (requires key export)"
-}
 
 variable "database_name" {
   type        = string
@@ -30,6 +25,18 @@ variable "datastore_s3_bucket_kms_key_arn" {
   description = "The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket"
 }
 
+variable "enable_api_basic_auth" {
+  type        = bool
+  default     = true
+  description = "Enable basic auth for API Gateway? (requires key export)"
+}
+
+variable "enable_api_gateway" {
+  type        = bool
+  default     = true
+  description = "Enable API Gateway for public metadata service endpoint"
+}
+
 variable "fargate_execution_role_arn" {
   type        = string
   description = "The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution_role for our Fargate instance"
@@ -104,7 +111,6 @@ variable "subnet2_id" {
   type        = string
   description = "Second private subnet used for availability zone redundancy"
 }
-
 variable "vpc_cidr_blocks" {
   type        = list(string)
   description = "The VPC CIDR blocks that we'll access list on our Metadata Service API to allow all internal communications"

outputs.tf

@@ -74,7 +74,7 @@ output "metaflow_profile_json" {
         "METAFLOW_BATCH_CONTAINER_REGISTRY" = element(split("/", aws_ecr_repository.metaflow_batch_image[0].repository_url), 0),
         "METAFLOW_BATCH_CONTAINER_IMAGE"    = element(split("/", aws_ecr_repository.metaflow_batch_image[0].repository_url), 1)
       } : {},
-      var.api_basic_auth ? {
+      var.metadata_service_enable_api_basic_auth ? {
         "METAFLOW_SERVICE_AUTH_KEY" = "## Replace with output from 'aws apigateway get-api-key --api-key ${module.metaflow-metadata-service.api_gateway_rest_api_id_key_id} --include-value | grep value' ##"
       } : {},
       var.batch_type == "fargate" ? {

variables.tf

@@ -4,12 +4,6 @@ variable "access_list_cidr_blocks" {
   default     = []
 }
 
-variable "api_basic_auth" {
-  type        = bool
-  default     = true
-  description = "Enable basic auth for API Gateway? (requires key export)"
-}
-
 variable "batch_type" {
   type        = string
   description = "AWS Batch Compute Type ('ec2', 'fargate')"
@@ -97,6 +91,18 @@ variable "metadata_service_container_image" {
   description = "Container image for metadata service"
 }
 
+variable "metadata_service_enable_api_basic_auth" {
+  type        = bool
+  default     = true
+  description = "Enable basic auth for API Gateway? (requires key export)"
+}
+
+variable "metadata_service_enable_api_gateway" {
+  type        = bool
+  default     = true
+  description = "Enable API Gateway for public metadata service endpoint"
+}
+
 variable "ui_static_container_image" {
   type        = string
   default     = ""