diff --git a/terraform/aws/implementation/main.tf b/terraform/aws/implementation/main.tf index b2a5923f..bade62c5 100644 --- a/terraform/aws/implementation/main.tf +++ b/terraform/aws/implementation/main.tf @@ -1,39 +1,38 @@ locals { - name_prefix = "phdi-playground-${terraform.workspace}" - vpc_name = "${local.name_prefix}-vpc" - eks_name = "${local.name_prefix}-eks" + name = "phdi-playground-${terraform.workspace}" } module "vpc" { source = "terraform-aws-modules/vpc/aws" - name = local.vpc_name + name = local.name cidr = "176.24.0.0/16" azs = ["us-east-1a", "us-east-1b"] private_subnets = ["176.24.1.0/24", "176.24.3.0/24"] public_subnets = ["176.24.2.0/24", "176.24.4.0/24"] enable_nat_gateway = true single_nat_gateway = true - default_security_group_name = "${local.vpc_name}-security-group" + default_security_group_name = "${local.name}-security-group" public_subnet_tags = { - "kubernetes.io/cluster/${local.eks_name}" = "shared" - "kubernetes.io/role/elb" = "1" + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = "1" } private_subnet_tags = { - "kubernetes.io/cluster/${local.eks_name}" = "shared" - "kubernetes.io/role/internal-elb" = "1" + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + "karpenter.sh/discovery" = local.name } tags = { - "kubernetes.io/cluster/${local.eks_name}" = "shared" + "kubernetes.io/cluster/${local.name}" = "shared" } } module "eks" { source = "./modules/eks" region = var.region - eks_name = local.eks_name + eks_name = local.name vpc_id = module.vpc.vpc_id public_subnet_ids = module.vpc.public_subnets private_subnet_ids = module.vpc.private_subnets @@ -47,4 +46,4 @@ module "cloudfront" { region = var.region vpc_id = module.vpc.vpc_id alb_hostname = module.eks.alb_hostname -} \ No newline at end of file +} diff --git a/terraform/aws/implementation/modules/eks/data.tf b/terraform/aws/implementation/modules/eks/data.tf index cde1da44..66d8e18e 100644 --- a/terraform/aws/implementation/modules/eks/data.tf +++ b/terraform/aws/implementation/modules/eks/data.tf @@ -351,3 +351,7 @@ data "kubernetes_ingress_v1" "ingress" { } } } + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws +} diff --git a/terraform/aws/implementation/modules/eks/main.tf b/terraform/aws/implementation/modules/eks/main.tf index 28168eb1..6af7e0cd 100644 --- a/terraform/aws/implementation/modules/eks/main.tf +++ b/terraform/aws/implementation/modules/eks/main.tf @@ -21,41 +21,48 @@ module "eks-cluster" { vpc_id = var.vpc_id subnet_ids = flatten([var.private_subnet_ids, var.public_subnet_ids]) - cluster_addons = { - coredns = { - most_recent = true - } - kube-proxy = { - most_recent = true - } - vpc-cni = { - most_recent = true - } - } + # Fargate profiles use the cluster primary security group so these are not utilized + create_cluster_security_group = false + create_node_security_group = false fargate_profiles = { default = { - name = "default" selectors = [ - { - namespace = "kube-system" - }, - { - namespace = "default" - } + { namespace = "default" } + ] + subnet_ids = var.private_subnet_ids + } + karpenter = { + selectors = [ + { namespace = "karpenter" } + ] + subnet_ids = var.private_subnet_ids + } + kube_system = { + name = "kube-system" + selectors = [ + { namespace = "kube-system" } ] subnet_ids = var.private_subnet_ids } } manage_aws_auth_configmap = true - aws_auth_roles = [ { rolearn = "arn:aws:iam::339712971032:role/Developer" username = "Developer" groups = ["system:masters"] - } + }, + # We need to add in the Karpenter node IAM role for nodes launched by Karpenter + { + rolearn = module.eks_blueprints_addons.karpenter.node_iam_role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, ] aws_auth_users = [ @@ -69,6 +76,10 @@ module "eks-cluster" { aws_auth_accounts = [ "339712971032" ] + + tags = { + "karpenter.sh/discovery" = var.eks_name + } } data "aws_eks_cluster_auth" "eks" { @@ -115,6 +126,10 @@ locals { oidc_provider = module.eks-cluster.oidc_provider namespace = "kube-system" service_account_name = "aws-load-balancer-controller" + tags = { + Blueprint = var.eks_name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } } resource "aws_iam_role" "eks_service_account" { @@ -275,3 +290,60 @@ resource "kubectl_manifest" "ingress" { depends_on = [helm_release.building_blocks] yaml_body = data.kubectl_file_documents.ingress.documents[0] } + +# Karpenter + +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.14" + + cluster_name = module.eks-cluster.cluster_name + cluster_endpoint = module.eks-cluster.cluster_endpoint + cluster_version = module.eks-cluster.cluster_version + oidc_provider_arn = module.eks-cluster.oidc_provider_arn + + # We want to wait for the Fargate profiles to be deployed first + create_delay_dependencies = [for prof in module.eks-cluster.fargate_profiles : prof.fargate_profile_arn] + + eks_addons = { + coredns = { + configuration_values = jsonencode({ + computeType = "Fargate" + # Ensure that the we fully utilize the minimum amount of resources that are supplied by + # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html + # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes + # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following + # compute configuration that most closely matches the sum of vCPU and memory requests in + # order to ensure pods always have the resources that they need to run. + resources = { + limits = { + cpu = "0.25" + # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the + # request/limit to ensure we can fit within that task + memory = "256M" + } + requests = { + cpu = "0.25" + # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the + # request/limit to ensure we can fit within that task + memory = "256M" + } + } + }) + } + vpc-cni = {} + kube-proxy = {} + } + + enable_karpenter = true + karpenter = { + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password + } + karpenter_node = { + # Use static name so that it matches what is defined in `karpenter.yaml` example manifest + iam_role_use_name_prefix = false + } + + tags = local.tags +}