diff --git a/README.md b/README.md index 019fe033..8f9c136e 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,50 @@ tEKS is a set of Terraform / Terragrunt modules designed to get you everything you need to run a production EKS cluster on AWS. It ships with sensible defaults, and add a lot of common addons with their configurations that work out of the box. -:warning: the v5 of this project has been completely revamp and now offer a skeleton to use as a base for your infrastructure projects around EKS. All the modules have been moved outside this repository and get their own versioning. The [old README is accessible here](https://github.com/clusterfrak-dynamics/teks/tree/release-4.X) +:warning: the v5 and further version of this project have been completely revamp and now offer a skeleton to use as a base for your infrastructure projects around EKS. All the modules have been moved outside this repository and get their own versioning. The [old README is accessible here](https://github.com/clusterfrak-dynamics/teks/tree/release-4.X) -## Modules +## Main purposes -* [`terraform-aws-vpc`](https://github.com/terraform-aws-modules/terraform-aws-vpc) -* [`terraform-aws-eks`](https://github.com/terraform-aws-modules/terraform-aws-eks) -* [`terraform-kubernetes-addons`](https://github.com/clusterfrak-dynamics/terraform-kubernetes-addons): provides various addons that are often used on Kubernetes and specifically on EKS. -* [`terraform-kubernetes-namespaces`](https://github.com/clusterfrak-dynamics/terraform-kubernetes-addons): allows administrator to manage namespaces and quotas from a centralized configuration with Terraform. +The main goal of this project is to glue together commonly used tooling with Kubernetes/EKS and to get from an AWS Account to a production cluster with everything you need without any manual configuration. + +## What you get + +A production cluster all defined in IaaC with Terraform/Terragrunt: + +* AWS VPC if needed based on [`terraform-aws-vpc`](https://github.com/terraform-aws-modules/terraform-aws-vpc) +* EKS cluster base on [`terraform-aws-eks`](https://github.com/terraform-aws-modules/terraform-aws-eks) +* Kubernetes addons based on [`terraform-kubernetes-addons`](https://github.com/clusterfrak-dynamics/terraform-kubernetes-addons): provides various addons that are often used on Kubernetes and specifically on EKS. +* Kubernetes namespaces quota management based on [`terraform-kubernetes-namespaces`](https://github.com/clusterfrak-dynamics/terraform-kubernetes-addons): allows administrator to manage namespaces and quotas from a centralized configuration with Terraform. +* AWS ECR registries management based on [`terraform-aws-ecr`](https://github.com/clusterfrak-dynamics/terraform-aws-ecr) + +Everything is tied together with Terragrunt and allows you to deploy a multi cluster architecture in a matter of minutes (ok maybe an hour) and different AWS accounts for different environments. + +## Curated Features + +The main additionals features are the curated addons list, see [here](https://github.com/clusterfrak-dynamics/terraform-kubernetes-addons) and in the customization of the cluster policy + +### Enforced security + +* Default PSP is removed and sensible defaults are enforced +* All addons have specific PSP enabled +* No IAM credentials on instances, everything is enforced with [IRSA](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/) or [KIAM](https://github.com/uswitch/kiam) + +### Out of the box monitoring + +* Prometheus Operator with defaults dashboards +* Addons that support metrics are enable along with their `serviceMonitor` +* Custom grafana dashboard are available by default. + +### Helm v3 provider + +* All addons support Helm v3 configuration +* All charts are easily customizable + +### Other and not limited to + +* priorityClasses for addons +* use of [`kubectl-provider`], no more local exec and custom manifest are properly handled +* lot of manual stuff have been automated under the hood ## Requirements diff --git a/requirements.txt b/requirements.txt index 95bb61fd..4c944ccc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ # Linux -https://releases.hashicorp.com/terraform/0.11.14/terraform_0.12.17_linux_amd64.zip -https://github.com/gruntwork-io/terragrunt/releases/download/v0.21.6/terragrunt_linux_amd64 +https://releases.hashicorp.com/terraform/0.12.21/terraform_0.12.21_linux_amd64.zip +https://github.com/gruntwork-io/terragrunt/releases/download/v0.22.3/terragrunt_linux_amd64 https://storage.googleapis.com/kubernetes-release/release/v1.14.8/bin/linux/amd64/kubectl -https://get.helm.sh/helm-v2.16.1-linux-amd64.tar.gz +https://get.helm.sh/helm-v3.1.0-linux-amd64.tar.gz https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.4.0/aws-iam-authenticator_0.4.0_linux_amd64 # Darwin -https://releases.hashicorp.com/terraform/0.11.14/terraform_0.12.17_darwin_amd64.zip -https://github.com/gruntwork-io/terragrunt/releases/download/v0.21.6/terragrunt_darwin_amd64 +https://releases.hashicorp.com/terraform/0.12.21/terraform_0.12.21_darwin_amd64.zip +https://github.com/gruntwork-io/terragrunt/releases/download/v0.22.3/terragrunt_linux_amd64 https://storage.googleapis.com/kubernetes-release/release/v1.14.8/bin/darwin/amd64/kubectl -https://get.helm.sh/helm-v2.16.1-darwin-amd64.tar.gz +https://get.helm.sh/helm-v3.1.0-darwin-amd64.tar.gz https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.4.0/aws-iam-authenticator_0.4.0_darwin_amd64 diff --git a/terraform/live/demo/eu-west-3/ecr/provider.tf b/terraform/live/demo/eu-west-3/ecr/provider.tf index 16571418..826930db 100644 --- a/terraform/live/demo/eu-west-3/ecr/provider.tf +++ b/terraform/live/demo/eu-west-3/ecr/provider.tf @@ -4,7 +4,7 @@ terraform { } provider "aws" { - region = var.aws["region"] + region = var.aws["region"] version = "~> 2.41" } diff --git a/terraform/live/demo/eu-west-3/ecr/terragrunt.hcl b/terraform/live/demo/eu-west-3/ecr/terragrunt.hcl index 2f68c3b4..2f0df02e 100644 --- a/terraform/live/demo/eu-west-3/ecr/terragrunt.hcl +++ b/terraform/live/demo/eu-west-3/ecr/terragrunt.hcl @@ -3,7 +3,7 @@ include { } terraform { - source = "github.com/clusterfrak-dynamics/terraform-aws-ecr.git?ref=v2.0.1" + source = "github.com/clusterfrak-dynamics/terraform-aws-ecr.git?ref=v2.1.0" } locals { diff --git a/terraform/live/demo/eu-west-3/eks-addons/examples/keycloak-values.yaml b/terraform/live/demo/eu-west-3/eks-addons/examples/keycloak-values.yaml new file mode 100644 index 00000000..5bcdd7dc --- /dev/null +++ b/terraform/live/demo/eu-west-3/eks-addons/examples/keycloak-values.yaml @@ -0,0 +1,20 @@ +--- +keycloak: + username: admin + ingress: + enabled: true + path: / + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + hosts: + - keycloak.${local.env}.synthesis-care.com + persistence: + deployPostgres: false + dbVendor: postgres + existingSecret: "db-keycloak-${local.env}" + existingSecretKey: "DB_PASSWORD" + dbName: keycloak + dbHost: ${dependency.keycloak.outputs.db_instance_address} + dbPort: 5432 + dbUser: keycloak diff --git a/terraform/live/demo/eu-west-3/eks-addons/examples/kong-values.yaml b/terraform/live/demo/eu-west-3/eks-addons/examples/kong-values.yaml new file mode 100644 index 00000000..f0a666a4 --- /dev/null +++ b/terraform/live/demo/eu-west-3/eks-addons/examples/kong-values.yaml @@ -0,0 +1,25 @@ +--- +serviceMonitor: + enabled: true +env: + REAL_IP_HEADER: "proxy-protocol" + CLIENT_BODY_BUFFER_SIZE: "100M" + CUSTOM_PLUGINS: "oidc" + PLUGINS: "oidc,cors,prometheus" + NGINX_SEND_TIMEOUT: 600 + NGINX_MAX_EXECUTION_TIME: 600 + NGINX_REQUEST_TERMINATE_TIMEOUT: 600 +proxy: + type: LoadBalancer + http: + servicePort: 443 + tls: + enabled: false + servicePort: 443 + containerPort: 8000 + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "${dependency.acm.outputs.certificate_arn}" + external-dns.alpha.kubernetes.io/hostname: "kong.${local.env}.cfd.io" + externalTrafficPolicy: "Cluster" diff --git a/terraform/live/demo/eu-west-3/eks-addons/providers.tf b/terraform/live/demo/eu-west-3/eks-addons/providers.tf index bf4e0cb3..830fbe35 100644 --- a/terraform/live/demo/eu-west-3/eks-addons/providers.tf +++ b/terraform/live/demo/eu-west-3/eks-addons/providers.tf @@ -4,24 +4,27 @@ terraform { } provider "aws" { - region = var.aws["region"] + region = var.aws["region"] version = "~> 2.41" } +provider "kubectl" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false +} + provider "kubernetes" { host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) token = data.aws_eks_cluster_auth.cluster.token load_config_file = false - version = "~> 1.9" + version = "1.10" } provider "helm" { - install_tiller = true - service_account = "tiller" - tiller_image = "gcr.io/kubernetes-helm/tiller:v2.16.1" - automount_service_account_token = true - + version = "~> 1.0" kubernetes { host = data.aws_eks_cluster.cluster.endpoint cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) @@ -37,9 +40,9 @@ data "aws_caller_identity" "current" { } data "aws_eks_cluster" "cluster" { - name = var.eks["cluster_name"] + name = var.cluster-name } data "aws_eks_cluster_auth" "cluster" { - name = var.eks["cluster_name"] + name = var.cluster-name } diff --git a/terraform/live/demo/eu-west-3/eks-addons/terragrunt.hcl b/terraform/live/demo/eu-west-3/eks-addons/terragrunt.hcl index f8ff059a..10a3076e 100644 --- a/terraform/live/demo/eu-west-3/eks-addons/terragrunt.hcl +++ b/terraform/live/demo/eu-west-3/eks-addons/terragrunt.hcl @@ -3,7 +3,12 @@ include { } terraform { - source = "github.com/clusterfrak-dynamics/terraform-kubernetes-addons.git?ref=v4.0.1" + source = "github.com/clusterfrak-dynamics/terraform-kubernetes-addons.git?ref=v5.0.0" + + before_hook "init" { + commands = ["init"] + execute = ["bash", "-c", "wget -O terraform-provider-kubectl https://github.com/gavinbunney/terraform-provider-kubectl/releases/download/v1.2.1/terraform-provider-kubectl-linux-amd64 && chmod +x terraform-provider-kubectl"] + } } locals { @@ -15,7 +20,19 @@ dependency "eks" { config_path = "../eks" mock_outputs = { - cluster_id = "cluster-name" + cluster_id = "cluster-name" + cluster_oidc_issuer_url = "https://oidc.eks.eu-west-3.amazonaws.com/id/0000000000000000" + } +} + +dependency "vpc" { + config_path = "../vpc" + + mock_outputs = { + private_subnets_cidr_blocks = [ + "10.0.0.0/16", + "192.168.0.0/24" + ] } } @@ -28,275 +45,116 @@ inputs = { } eks = { - "cluster_name" = dependency.eks.outputs.cluster_id + "cluster_oidc_issuer_url" = dependency.eks.outputs.cluster_oidc_issuer_url } nginx_ingress = { - version = "0.26.1" - chart_version = "1.24.5" - enabled = true + version = "0.29.0" + chart_version = "1.31.0" + enabled = false default_network_policy = true ingress_cidr = "0.0.0.0/0" - namespace = "ingress-nginx" - timeout = 3600 - force_update = false - recreate_pods = false - wait = true - - extra_values = </dev/null > ${get_terragrunt_dir()}/kubeconfig"] } + + after_hook "kube-system-label" { + commands = ["apply"] + execute = ["bash", "-c", "kubectl --kubeconfig kubeconfig label ns kube-system name=kube-system --overwrite"] + } + + after_hook "remove-default-psp" { + commands = ["apply"] + execute = ["bash", "-c", "kubectl --kubeconfig kubeconfig delete psp eks.privileged || true"] + } + after_hook "remove-default-psp-clusterrolebindind" { + commands = ["apply"] + execute = ["bash", "-c", "kubectl --kubeconfig kubeconfig delete clusterrolebinding eks:podsecuritypolicy:authenticated || true"] + } + after_hook "remove-default-psp-clusterrole" { + commands = ["apply"] + execute = ["bash", "-c", "kubectl --kubeconfig kubeconfig delete clusterrole eks:podsecuritypolicy:privileged || true"] + } } locals { @@ -44,10 +62,12 @@ inputs = { "region" = local.aws_region } + psp_privileged_ns = [ + "cluster-autoscaler", #waiting for https://github.com/helm/charts/pull/20891 + "istio-system" #istio does not support psp by default + ] + tags = merge( - { - "kubernetes.io/cluster/${local.cluster_name}" = "shared" - }, local.custom_tags ) @@ -85,9 +105,9 @@ inputs = { { name = "default-${local.aws_region}b" instance_type = "t3.medium" - asg_min_size = 1 + asg_min_size = 0 asg_max_size = 3 - asg_desired_capacity = 1 + asg_desired_capacity = 0 subnets = [dependency.vpc.outputs.private_subnets[1]] autoscaling_enabled = true root_volume_size = 50 @@ -102,9 +122,9 @@ inputs = { { name = "default-${local.aws_region}c" instance_type = "t3.medium" - asg_min_size = 1 + asg_min_size = 0 asg_max_size = 3 - asg_desired_capacity = 1 + asg_desired_capacity = 0 subnets = [dependency.vpc.outputs.private_subnets[2]] autoscaling_enabled = true root_volume_size = 50 @@ -117,6 +137,4 @@ inputs = { ] }, ] - } - diff --git a/terraform/live/demo/eu-west-3/kiam-roles/terragrunt.hcl b/terraform/live/demo/eu-west-3/kiam-roles/terragrunt.hcl deleted file mode 100644 index d206bec3..00000000 --- a/terraform/live/demo/eu-west-3/kiam-roles/terragrunt.hcl +++ /dev/null @@ -1,77 +0,0 @@ -include { - path = "${find_in_parent_folders()}" -} - -terraform { - source = "github.com/clusterfrak-dynamics/terraform-aws-iam-roles?ref=v1.0.1" -} - -locals { - aws_region = yamldecode(file("${get_terragrunt_dir()}/${find_in_parent_folders("common_values.yaml")}"))["aws_region"] - env = yamldecode(file("${get_terragrunt_dir()}/${find_in_parent_folders("common_tags.yaml")}"))["Env"] - custom_tags = yamldecode(file("${get_terragrunt_dir()}/${find_in_parent_folders("common_tags.yaml")}")) - prefix = yamldecode(file("${get_terragrunt_dir()}/${find_in_parent_folders("common_values.yaml")}"))["prefix"] -} - -dependency "eks-addons" { - config_path = "../eks-addons" - - mock_outputs = { - kiam-server-role-arn = ["arn:aws:iam::000000000000:role/mock-role"] - } -} - -inputs = { - - env = local.env - - aws = { - "region" = local.aws_region - } - - iam_roles = [ - { - name = "${local.prefix}-s3-access-${local.env}" - policy = <