Add support for externalized compute clusters to cdp_deploy module

modules/terraform-cdp-deploy/README.md

@@ -87,6 +87,8 @@ No resources.
 | <a name="input_azure_xaccount_app_pword"></a> [azure\_xaccount\_app\_pword](#input\_azure\_xaccount\_app\_pword) | Password for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_azure_xaccount_app_uuid"></a> [azure\_xaccount\_app\_uuid](#input\_azure\_xaccount\_app\_uuid) | UUID for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_cdp_xacccount_credential_name"></a> [cdp\_xacccount\_credential\_name](#input\_cdp\_xacccount\_credential\_name) | Name of the CDP Cross Account Credential. Defaults to '<env\_prefix>-xaccount-cred' if not specified. If create\_cdp\_credential is set to false then this should should be a valid pre-existing credential. | `string` | `null` | no |
+| <a name="input_compute_cluster_configuration"></a> [compute\_cluster\_configuration](#input\_compute\_cluster\_configuration) | Kubernetes configuration for the externalized compute cluster | <pre>object({<br>    kube_api_authorized_ip_ranges = optional(set(string))<br>    outbound_type                 = optional(string)<br>    private_cluster               = optional(bool)<br>    worker_node_subnets           = optional(set(string))<br>  })</pre> | `null` | no |
+| <a name="input_compute_cluster_enabled"></a> [compute\_cluster\_enabled](#input\_compute\_cluster\_enabled) | Enable externalized compute cluster for the environment | `bool` | `false` | no |
 | <a name="input_create_cdp_credential"></a> [create\_cdp\_credential](#input\_create\_cdp\_credential) | Flag to specify if the CDP Cross Account Credential should be created. If set to false then cdp\_xacccount\_credential\_name should be a valid pre-existing credential. | `bool` | `true` | no |
 | <a name="input_datalake_async_creation"></a> [datalake\_async\_creation](#input\_datalake\_async\_creation) | Flag to specify if Terraform should wait for CDP datalake resource creation/deletion | `bool` | `false` | no |
 | <a name="input_datalake_call_failure_threshold"></a> [datalake\_call\_failure\_threshold](#input\_datalake\_call\_failure\_threshold) | Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling | `number` | `3` | no |

modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf

@@ -40,6 +40,11 @@ terraform {
 
 provider "aws" {
   region = var.aws_region
+
+  # ignore tags created by data services
+  ignore_tags {
+    key_prefixes = ["kubernetes.io/cluster"]
+  }
 }
 
 module "cdp_aws_prereqs" {
@@ -97,6 +102,9 @@ module "cdp_deploy" {
   environment_async_creation = var.environment_async_creation
   datalake_async_creation    = var.datalake_async_creation
 
+  compute_cluster_enabled       = var.compute_cluster_enabled
+  compute_cluster_configuration = var.compute_cluster_configuration
+
   # From pre-reqs module output
   aws_vpc_id             = module.cdp_aws_prereqs.aws_vpc_id
   aws_public_subnet_ids  = module.cdp_aws_prereqs.aws_public_subnet_ids

modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf

@@ -97,6 +97,26 @@ variable "cdp_groups" {
 
   default = null
 }
+variable "compute_cluster_enabled" {
+  type = bool
+
+  description = "Enable externalized compute cluster for the environment"
+
+  default = false
+}
+
+variable "compute_cluster_configuration" {
+  type = map(object({
+    kube_api_authorized_ip_ranges = optional(set(string))
+    private_cluster               = optional(bool)
+    worker_node_subnets           = optional(set(string))
+  }))
+
+  description = "Kubernetes configuration for the externalized compute cluster"
+
+  default = null
+}
+
 # ------- Network Resources -------
 variable "ingress_extra_cidrs_and_ports" {
   type = object({

modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf

@@ -88,6 +88,9 @@ module "cdp_deploy" {
   environment_async_creation = var.environment_async_creation
   datalake_async_creation    = var.datalake_async_creation
 
+  compute_cluster_enabled       = var.compute_cluster_enabled
+  compute_cluster_configuration = var.compute_cluster_configuration
+
   # From pre-reqs module output
   azure_subscription_id = module.cdp_azure_prereqs.azure_subscription_id
   azure_tenant_id       = module.cdp_azure_prereqs.azure_tenant_id

modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf

@@ -93,6 +93,24 @@ variable "cdp_groups" {
     ]))
     error_message = "CDP group names can consist only of letters, numbers, dots (.), hyphens (-) and underscores (_)."
   }
+}
+variable "compute_cluster_enabled" {
+  type = bool
+
+  description = "Enable externalized compute cluster for the environment"
+
+  default = false
+}
+
+variable "compute_cluster_configuration" {
+  type = map(object({
+    kube_api_authorized_ip_ranges = optional(set(string))
+    outbound_type                 = optional(string)
+    private_cluster               = optional(bool)
+    worker_node_subnets           = optional(set(string))
+  }))
+
+  description = "Kubernetes configuration for the externalized compute cluster"
 
   default = null
 }

modules/terraform-cdp-deploy/main.tf

@@ -85,6 +85,9 @@ module "cdp_on_aws" {
   datalake_recipes      = var.datalake_recipes
 
   environment_cascading_delete = var.environment_cascading_delete
+
+  compute_cluster_enabled       = var.compute_cluster_enabled
+  compute_cluster_configuration = var.compute_cluster_configuration
 }
 
 # ------- Call sub-module for Azure Deployment -------
@@ -177,6 +180,9 @@ module "cdp_on_azure" {
   datalake_flexible_server_delegated_subnet_name = var.azure_datalake_flexible_server_delegated_subnet_name
 
   environment_cascading_delete = var.environment_cascading_delete
+
+  compute_cluster_enabled       = var.compute_cluster_enabled
+  compute_cluster_configuration = var.compute_cluster_configuration
 }
 
 # ------- Call sub-module for GCP Deployment -------

modules/terraform-cdp-deploy/modules/aws/main.tf

@@ -61,6 +61,11 @@ resource "cdp_environments_aws_environment" "cdp_env" {
     os                      = var.freeipa_os
   }
 
+  compute_cluster = {
+    enabled       = var.compute_cluster_enabled
+    configuration = var.compute_cluster_configuration
+  }
+
   proxy_config_name   = var.proxy_config_name
   s3_guard_table_name = var.s3_guard_table_name
   workload_analytics  = var.workload_analytics

modules/terraform-cdp-deploy/modules/aws/variables.tf

@@ -178,6 +178,23 @@ variable "workload_analytics" {
 
 }
 
+variable "compute_cluster_enabled" {
+  type = bool
+
+  description = "Enable externalized compute cluster for the environment"
+
+}
+
+variable "compute_cluster_configuration" {
+  type = object({
+    kube_api_authorized_ip_ranges = optional(set(string))
+    private_cluster               = optional(bool)
+    worker_node_subnets           = optional(set(string))
+  })
+
+  description = "Kubernetes configuration for the externalized compute cluster"
+}
+
 variable "datalake_scale" {
   type = string
 

modules/terraform-cdp-deploy/modules/azure/main.tf

@@ -82,6 +82,11 @@ resource "cdp_environments_azure_environment" "cdp_env" {
     os                      = var.freeipa_os
   }
 
+  compute_cluster = {
+    enabled       = var.compute_cluster_enabled
+    configuration = var.compute_cluster_configuration
+  }
+
   proxy_config_name  = var.proxy_config_name
   workload_analytics = var.workload_analytics
   enable_tunnel      = var.enable_ccm_tunnel

modules/terraform-cdp-deploy/modules/azure/variables.tf

@@ -212,6 +212,23 @@ variable "proxy_config_name" {
 
 }
 
+variable "compute_cluster_enabled" {
+  type = bool
+
+  description = "Enable externalized compute cluster for the environment"
+
+}
+
+variable "compute_cluster_configuration" {
+  type = object({
+    kube_api_authorized_ip_ranges = optional(set(string))
+    outbound_type                 = optional(string)
+    private_cluster               = optional(bool)
+    worker_node_subnets           = optional(set(string))
+  })
+
+  description = "Kubernetes configuration for the externalized compute cluster"
+}
 
 variable "datalake_scale" {
   type = string

modules/terraform-cdp-deploy/variables.tf

@@ -280,6 +280,27 @@ variable "workload_analytics" {
   default = true
 }
 
+variable "compute_cluster_enabled" {
+  type = bool
+
+  description = "Enable externalized compute cluster for the environment"
+
+  default = false
+}
+
+variable "compute_cluster_configuration" {
+  type = object({
+    kube_api_authorized_ip_ranges = optional(set(string))
+    outbound_type                 = optional(string)
+    private_cluster               = optional(bool)
+    worker_node_subnets           = optional(set(string))
+  })
+
+  description = "Kubernetes configuration for the externalized compute cluster"
+
+  default = null
+}
+
 variable "datalake_scale" {
   type = string