Merge pull request #36 from walkline/blueprints-v5

Migrate to aws-eks-blueprints v5

Author: tbulding

.header.md

@@ -140,7 +140,7 @@ scales up and down by allowing Kubernetes to modify the Amazon EC2 Auto Scaling
 
 A basic logging and monitoring stack containing Prometheus, Grafana, Loki and Promtail is available at the following URL: `https://monitoring.{domain_name}`
 
-To retrieve the Grafana administrative credentials, run the following command: 
+To retrieve the Grafana administrative credentials (with `admin` username), run the following command: 
 
 ```
 terraform output -json grafana_admin_password
@@ -208,10 +208,9 @@ Otherwise, you will need to manually remove some finalizers in the namespace and
 To clean up your environment, run the following commands:
 
 ```
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx[0].module.helm_addon.helm_release.addon[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx[0].kubernetes_namespace_v1.this[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.prometheus[0].module.helm_addon.helm_release.addon[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.prometheus[0].kubernetes_namespace_v1.prometheus[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx.helm_release.this[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.kube_prometheus_stack.helm_release.this[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.aws_load_balancer_controller.helm_release.this[0]" -auto-approve
 terraform destroy -target="module.eks_blueprints_kubernetes_addons" -auto-approve
 terraform destroy -auto-approve
 ```

README.md

@@ -141,7 +141,7 @@ scales up and down by allowing Kubernetes to modify the Amazon EC2 Auto Scaling
 
 A basic logging and monitoring stack containing Prometheus, Grafana, Loki and Promtail is available at the following URL: `https://monitoring.{domain_name}`
 
-To retrieve the Grafana administrative credentials, run the following command:
+To retrieve the Grafana administrative credentials (with `admin` username), run the following command:
 
 ```
 terraform output -json grafana_admin_password
@@ -209,10 +209,9 @@ Otherwise, you will need to manually remove some finalizers in the namespace and
 To clean up your environment, run the following commands:
 
 ```
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx[0].module.helm_addon.helm_release.addon[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx[0].kubernetes_namespace_v1.this[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.prometheus[0].module.helm_addon.helm_release.addon[0]" -auto-approve
-terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.prometheus[0].kubernetes_namespace_v1.prometheus[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.ingress_nginx.helm_release.this[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.kube_prometheus_stack.helm_release.this[0]" -auto-approve
+terraform destroy -target="module.eks_blueprints_kubernetes_addons.module.aws_load_balancer_controller.helm_release.this[0]" -auto-approve
 terraform destroy -target="module.eks_blueprints_kubernetes_addons" -auto-approve
 terraform destroy -auto-approve
 ```
@@ -252,8 +251,9 @@ After you deploy this Partner Solution, confirm that your resources and services
 |------|--------|---------|
 | <a name="module_container_registry"></a> [container\_registry](#module\_container\_registry) | ./modules/container-registry | n/a |
 | <a name="module_databases"></a> [databases](#module\_databases) | ./modules/databases | n/a |
-| <a name="module_eks_blueprints"></a> [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.32.1 |
-| <a name="module_eks_blueprints_kubernetes_addons"></a> [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.32.1 |
+| <a name="module_ebs_csi_driver_irsa"></a> [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 |
+| <a name="module_eks_blueprints"></a> [eks\_blueprints](#module\_eks\_blueprints) | terraform-aws-modules/eks/aws | ~> 19.13 |
+| <a name="module_eks_blueprints_kubernetes_addons"></a> [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.8.0 |
 | <a name="module_file_storage"></a> [file\_storage](#module\_file\_storage) | ./modules/file-storage | n/a |
 | <a name="module_monitoring"></a> [monitoring](#module\_monitoring) | ./modules/monitoring | n/a |
 | <a name="module_vpc"></a> [vpc](#module\_vpc) | ./modules/vpc | n/a |
@@ -287,7 +287,7 @@ After you deploy this Partner Solution, confirm that your resources and services
 | <a name="input_eks_cluster_name_prefix"></a> [eks\_cluster\_name\_prefix](#input\_eks\_cluster\_name\_prefix) | EKS name prefix for the new cluster | `string` | `"mendix-eks"` | no |
 | <a name="input_eks_node_instance_type"></a> [eks\_node\_instance\_type](#input\_eks\_node\_instance\_type) | EKS instance type | `string` | `"t3.medium"` | no |
 | <a name="input_environments_internal_names"></a> [environments\_internal\_names](#input\_environments\_internal\_names) | List of internal environments names | `list(string)` | <pre>[<br>  "app1"<br>]</pre> | no |
-| <a name="input_mendix_operator_version"></a> [mendix\_operator\_version](#input\_mendix\_operator\_version) | Mendix Private Cloud Operator version | `string` | `"2.12.0"` | no |
+| <a name="input_mendix_operator_version"></a> [mendix\_operator\_version](#input\_mendix\_operator\_version) | Mendix Private Cloud Operator version | `string` | `"2.13.0"` | no |
 | <a name="input_postgres_version"></a> [postgres\_version](#input\_postgres\_version) | The version of Postgres that terraform would create. | `string` | `"14.8"` | no |
 
 ## Outputs

helm-values/prometheus-values.yaml

@@ -1,11 +1,66 @@
-server:
-  retention: 60d
-  persistentVolume:
-    enabled: true
-    size: 30Gi
+prometheus:
+  prometheusSpec:
+    additionalScrapeConfigs:
+    - job_name: kubernetes-pods
+      kubernetes_sd_configs:
+        - role: pod
+      relabel_configs:
+        - action: keep
+          regex: true
+          source_labels:
+            - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+        - action: drop
+          regex: true
+          source_labels:
+            - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
+        - action: replace
+          regex: (https?)
+          source_labels:
+            - __meta_kubernetes_pod_annotation_prometheus_io_scheme
+          target_label: __scheme__
+        - action: replace
+          regex: (.+)
+          source_labels:
+            - __meta_kubernetes_pod_annotation_prometheus_io_path
+          target_label: __metrics_path__
+        - action: replace
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          source_labels:
+            - __address__
+            - __meta_kubernetes_pod_annotation_prometheus_io_port
+          target_label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
+          replacement: __param_$1
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - action: replace
+          source_labels:
+            - __meta_kubernetes_namespace
+          target_label: namespace
+        - action: replace
+          source_labels:
+            - __meta_kubernetes_pod_name
+          target_label: pod
+        - action: drop
+          regex: Pending|Succeeded|Failed|Completed
+          source_labels:
+            - __meta_kubernetes_pod_phase
+    retention: 60d
+    storageSpec:
+      volumeClaimTemplate:
+        metadata:
+          name: prometheus-server
+        spec:
+          storageClassName: gp2
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 30Gi
 
-pushgateway:
-  enabled: false
-  
 alertmanager:
   enabled: false
+
+grafana:
+  enabled: false

main.tf

@@ -9,7 +9,7 @@ resource "random_string" "random_eks_suffix" {
 }
 
 data "aws_eks_cluster_auth" "this" {
-  name = module.eks_blueprints.eks_cluster_id
+  name = module.eks_blueprints.cluster_name
 }
 
 module "vpc" {
@@ -42,7 +42,7 @@ resource "aws_iam_policy" "environment_policy" {
   name        = "${local.cluster_name}-env-policy"
   description = "Environment Template Policy"
 
-  policy = templatefile("./iam-templates/iam_environment_policy.json.tpl", {
+  policy = templatefile("${path.module}/iam-templates/iam_environment_policy.json.tpl", {
     aws_region                     = var.aws_region
     aws_account_id                 = data.aws_caller_identity.current.account_id
     db_instance_resource_ids       = [for value in values(module.databases) : tostring(value.database_resource_id[0])]
@@ -54,7 +54,7 @@ resource "aws_iam_policy" "provisioner_policy" {
   name        = "${local.cluster_name}-provisioner-policy"
   description = "Storage Provisioner admin Policy"
 
-  policy = templatefile("./iam-templates/iam_provisioner_policy.json.tpl", {
+  policy = templatefile("${path.module}/iam-templates/iam_provisioner_policy.json.tpl", {
     aws_region                     = var.aws_region
     aws_account_id                 = data.aws_caller_identity.current.account_id
     db_instance_resource_ids       = [for value in values(module.databases) : tostring(value.database_resource_id[0])]
@@ -107,40 +107,31 @@ resource "aws_ebs_encryption_by_default" "ebs_encryption" {
 }
 
 module "eks_blueprints" {
-  source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.32.1"
-
-  node_security_group_additional_rules = {
-    cluster_to_nginx_webhook = {
-      description                   = "Cluster to ingress-nginx webhook"
-      protocol                      = "tcp"
-      from_port                     = 8443
-      to_port                       = 8443
-      type                          = "ingress"
-      source_cluster_security_group = true
-    }
-    cluster_to_load_balancer_controller_webhook = {
-      description                   = "Cluster to load balancer controller webhook"
-      protocol                      = "tcp"
-      from_port                     = 9443
-      to_port                       = 9443
-      type                          = "ingress"
-      source_cluster_security_group = true
-    }
-  }
+  source  = "terraform-aws-modules/eks/aws"
+  version = "~> 19.13"
 
   # EKS CLUSTER
-  cluster_name       = local.cluster_name
-  cluster_version    = "1.24"
-  vpc_id             = module.vpc.vpc_id
-  private_subnet_ids = module.vpc.vpc_private_subnets
+  cluster_name    = local.cluster_name
+  cluster_version = "1.26"
+  vpc_id          = module.vpc.vpc_id
+  subnet_ids      = module.vpc.vpc_private_subnets
 
   cluster_endpoint_public_access       = true
   cluster_endpoint_private_access      = true
   cluster_endpoint_public_access_cidrs = var.allowed_ips
 
+  create_node_security_group = false
+
   # EKS MANAGED NODE GROUPS
-  managed_node_groups = {
+  eks_managed_node_groups = {
     t3_medium = {
+      min_size     = 3
+      max_size     = 3
+      desired_size = 3
+
+      attach_cluster_primary_security_group = true
+      vpc_security_group_ids                = [module.eks_blueprints.cluster_primary_security_group_id]
+
       node_group_name = "managed-ondemand"
       instance_types  = [var.eks_node_instance_type]
       subnet_ids      = module.vpc.vpc_private_subnets
@@ -151,41 +142,52 @@ module "eks_blueprints" {
 }
 
 module "eks_blueprints_kubernetes_addons" {
-  source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1"
+  source  = "aws-ia/eks-blueprints-addons/aws"
+  version = "~> 1.8.0"
 
-  eks_cluster_id       = module.eks_blueprints.eks_cluster_id
-  eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint
-  eks_oidc_provider    = module.eks_blueprints.oidc_provider
-  eks_cluster_version  = module.eks_blueprints.eks_cluster_version
-  eks_cluster_domain   = var.domain_name
+  cluster_name      = module.eks_blueprints.cluster_name
+  cluster_endpoint  = module.eks_blueprints.cluster_endpoint
+  oidc_provider_arn = module.eks_blueprints.oidc_provider_arn
+  cluster_version   = module.eks_blueprints.cluster_version
 
   # EKS Managed Add-ons
-  enable_amazon_eks_coredns            = true
-  enable_amazon_eks_kube_proxy         = true
-  enable_amazon_eks_aws_ebs_csi_driver = true
+  eks_addons = {
+    coredns    = {}
+    kube-proxy = {}
+    aws-ebs-csi-driver = {
+      service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn
+    }
+  }
 
   # Add-ons
   enable_aws_load_balancer_controller = true
+  aws_load_balancer_controller = {
+    chart_version = "1.6.1"
+    set = [{
+      name  = "enableServiceMutatorWebhook"
+      value = "false"
+    }]
+  }
 
   enable_external_dns = true
   external_dns_route53_zone_arns = [
     aws_route53_zone.cluster_dns.arn
   ]
-  external_dns_helm_config = {
+  external_dns = {
     values = [templatefile("${path.module}/helm-values/external-dns-values.yaml", {
       hostname = var.domain_name
     })]
   }
 
   enable_ingress_nginx = true
-  ingress_nginx_helm_config = {
+  ingress_nginx = {
     values = [templatefile("${path.module}/helm-values/nginx-values.yaml", {
       hostname = var.domain_name
     })]
   }
 
   enable_cert_manager = true
-  cert_manager_helm_config = {
+  cert_manager = {
     set_values = [
       {
         name  = "extraArgs[0]"
@@ -194,9 +196,10 @@ module "eks_blueprints_kubernetes_addons" {
     ]
   }
 
-  enable_prometheus = true
-  prometheus_helm_config = {
-    values = [templatefile("${path.module}/helm-values/prometheus-values.yaml", {})]
+  enable_kube_prometheus_stack = true
+  kube_prometheus_stack = {
+    namespace = "prometheus"
+    values    = [templatefile("${path.module}/helm-values/prometheus-values.yaml", {})]
   }
 
   depends_on = [module.eks_blueprints, aws_route53_zone.cluster_dns]
@@ -251,4 +254,20 @@ resource "helm_release" "mendix_installer" {
   ]
 
   depends_on = [module.eks_blueprints, module.eks_blueprints_kubernetes_addons]
+}
+
+module "ebs_csi_driver_irsa" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "~> 5.20"
+
+  role_name_prefix = "${module.eks_blueprints.cluster_name}-ebs-csi-driver-"
+
+  attach_ebs_csi_policy = true
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks_blueprints.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+    }
+  }
 }

modules/monitoring/dashboards/pvc_disk_space.json.tpl

@@ -127,7 +127,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "kubelet_volume_stats_used_bytes{namespace=\"prometheus\",persistentvolumeclaim=\"prometheus-server\"}",
+          "expr": "kubelet_volume_stats_used_bytes{namespace=\"prometheus\",persistentvolumeclaim=\"prometheus-server-prometheus-kube-prometheus-stack-prometheus-0\"}",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -138,7 +138,7 @@
       ],
       "thresholds": [],
       "timeRegions": [],
-      "title": "disk space used (PVC \"prometheus-server\")",
+      "title": "disk space used (PVC \"prometheus-server-prometheus-kube-prometheus-stack-prometheus-0\")",
       "tooltip": {
         "shared": true,
         "sort": 0,

modules/monitoring/helm-values/grafana-values.yaml.tpl

@@ -33,7 +33,7 @@ datasources:
    datasources:
    - name: Prometheus
      type: prometheus
-     url: http://prometheus-server.prometheus.svc.cluster.local
+     url: http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090
      access: proxy
      isDefault: true
    - name: Loki

modules/vpc/main.tf

@@ -2,7 +2,7 @@ data "aws_availability_zones" "available" {}
 
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "4.0.1"
+  version = "5.1.2"
 
   name                 = "${var.cluster_name}-vpc"
   cidr                 = "10.0.0.0/16"

providers.tf

@@ -43,15 +43,15 @@ provider "aws" {
 }
 
 provider "kubernetes" {
-  host                   = module.eks_blueprints.eks_cluster_endpoint
-  cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data)
+  host                   = module.eks_blueprints.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks_blueprints.cluster_certificate_authority_data)
   token                  = data.aws_eks_cluster_auth.this.token
 }
 
 provider "helm" {
   kubernetes {
-    host                   = module.eks_blueprints.eks_cluster_endpoint
-    cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data)
+    host                   = module.eks_blueprints.cluster_endpoint
+    cluster_ca_certificate = base64decode(module.eks_blueprints.cluster_certificate_authority_data)
     token                  = data.aws_eks_cluster_auth.this.token
   }
 }