Begin to separate out the etcd cluster and the master clusters

io.tf

@@ -31,6 +31,7 @@ variable "instance-type" {
     bastion = "t2.nano"
     etcd = "c4.large"
     worker = "c4.large"
+    master = "c4.large"
   }
 }
 variable "internal-tld" {}
@@ -57,7 +58,7 @@ output "azs" { value = "${ var.aws["azs"] }" }
 output "bastion-ip" { value = "${ module.bastion.ip }" }
 output "dns-service-ip" { value = "${ var.dns-service-ip }" }
 output "etcd1-ip" { value = "${ element( split(",", var.etcd-ips), 0 ) }" }
-output "external-elb" { value = "${ module.etcd.external-elb }" }
+output "external-elb" { value = "${ module.master.external-elb }" }
 output "internal-tld" { value = "${ var.internal-tld }" }
 output "s3-bucket" { value = "${ var.s3-bucket }" }
 output "subnet-ids-private" { value = "${ module.vpc.subnet-ids-private }" }

modules.tf

@@ -8,6 +8,7 @@ module "s3" {
   internal-tld = "${ var.internal-tld }"
   name = "${ var.name }"
   region = "${ var.aws["region"] }"
+  service-ip-range = "${ var.cidr["service"] }"  
 }
 
 module "vpc" {
@@ -72,6 +73,32 @@ module "etcd" {
   vpc-id = "${ module.vpc.id }"
 }
 
+module "master" {
+  source = "./modules/master"
+  depends-id = "${ module.etcd.depends-id }"
+
+  ami-id = "${ var.coreos-aws["ami"] }"
+  bucket-prefix = "${ var.s3-bucket }"
+  coreos-hyperkube-image = "${ var.k8s["coreos-hyperkube-image"] }"
+  coreos-hyperkube-tag = "${ var.k8s["coreos-hyperkube-tag"] }"
+  dns-service-ip = "${ var.dns-service-ip }"
+  etcd-ips = "${ var.etcd-ips }"
+  master-security-group-id = "${ module.security.master-id }"
+  external-elb-security-group-id = "${ module.security.external-elb-id }"
+  instance-profile-name = "${ module.iam.instance-profile-name-master }"
+  instance-type = "${ var.instance-type["master"] }"
+  internal-tld = "${ var.internal-tld }"
+  key-name = "${ var.aws["key-name"] }"
+  name = "${ var.name }"
+  pod-ip-range = "${ var.cidr["pods"] }"
+  region = "${ var.aws["region"] }"
+  service-ip-range = "${ var.cidr["service"] }"
+  subnet-ids = "${ module.vpc.subnet-ids-public }"
+  vpc-cidr = "${ var.cidr["vpc"] }"
+  vpc-id = "${ module.vpc.id }"
+  internal-zone-id = "${ module.route53.internal-zone-id }"
+}
+
 module "bastion" {
   source = "./modules/bastion"
   depends-id = "${ module.etcd.depends-id }"
@@ -123,6 +150,6 @@ module "kubeconfig" {
   admin-key-pem = ".cfssl/k8s-admin-key.pem"
   admin-pem = ".cfssl/k8s-admin.pem"
   ca-pem = ".cfssl/ca.pem"
-  master-elb = "${ module.etcd.external-elb }"
+  master-elb = "${ module.master.external-elb }"
   name = "${ var.name }"
 }

modules/etcd/cloud-config.tf

@@ -9,34 +9,22 @@ coreos:
 
   etcd2:
     advertise-client-urls: http://${ fqdn }:2379
-    # cert-file: /etc/kubernetes/ssl/k8s-etcd.pem
+    # cert-file: /etc/etcd/ssl/k8s-etcd.pem
     # debug: true
     discovery-srv: ${ internal-tld }
     initial-advertise-peer-urls: https://${ fqdn }:2380
     initial-cluster-state: new
     initial-cluster-token: ${ cluster-token }
-    # key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem
+    # key-file: /etc/etcd/ssl/k8s-etcd-key.pem
     listen-client-urls: http://0.0.0.0:2379
     listen-peer-urls: https://0.0.0.0:2380
     name: ${ hostname }
-    peer-trusted-ca-file: /etc/kubernetes/ssl/ca.pem
+    peer-trusted-ca-file: /etc/etcd/ssl/ca.pem
     peer-client-cert-auth: true
-    peer-cert-file: /etc/kubernetes/ssl/k8s-etcd.pem
-    peer-key-file: /etc/kubernetes/ssl/k8s-etcd-key.pem
+    peer-cert-file: /etc/etcd/ssl/k8s-etcd.pem
+    peer-key-file: /etc/etcd/ssl/k8s-etcd-key.pem
 
   units:
-    - name: prefetch-hyperkube-container.service
-      command: start
-      content: |
-        [Unit]
-        Description=Accelerate spin up by prefetching hyperkube
-        After=network-online.target
-        [Service]
-        ExecStart=/usr/bin/rkt fetch --trust-keys-from-https \
-          ${ coreos-hyperkube-image }:${ coreos-hyperkube-tag }
-        RemainAfterExit=yes
-        Type=oneshot
-
     - name: etcd2.service
       command: start
       drop-ins:
@@ -46,33 +34,6 @@ coreos:
             After=get-ssl.service
             Requires=get-ssl.service
 
-    - name: flanneld.service
-      command: start
-      drop-ins:
-        - name: 50-network-config.conf
-          content: |
-            [Service]
-            ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config \
-              '{ "Network": "${ pod-ip-range }", "Backend": { "Type": "vxlan" } }'
-            Restart=always
-            RestartSec=10
-
-    - name: docker.service
-      command: start
-      drop-ins:
-        - name: 40-flannel.conf
-          content: |
-            [Unit]
-            After=flanneld.service
-            Requires=flanneld.service
-            [Service]
-            Restart=always
-            RestartSec=10
-        - name: overlay.conf
-          content: |
-            [Service]
-            Environment="DOCKER_OPTS=--storage-driver=overlay"
-
     - name: s3-get-presigned-url.service
       command: start
       content: |
@@ -96,9 +57,9 @@ coreos:
         Description=Get ssl artifacts from s3 bucket using IAM role
         Requires=s3-get-presigned-url.service
         [Service]
-        ExecStartPre=-/usr/bin/mkdir -p /etc/kubernetes/ssl
+        ExecStartPre=-/usr/bin/mkdir -p /etc/etcd/ssl
         ExecStart=/bin/sh -c "/usr/bin/curl $(/opt/bin/s3-get-presigned-url \
-          ${ region } ${ bucket } ${ ssl-tar }) | tar xv -C /etc/kubernetes/ssl/"
+          ${ region } ${ bucket } ${ ssl-tar }) | tar xv -C /etc/etcd/ssl/"
         RemainAfterExit=yes
         Type=oneshot
 
@@ -116,31 +77,6 @@ coreos:
         RemainAfterExit=yes
         Type=oneshot
 
-    - name: kubelet.service
-      command: start
-      content: |
-        [Unit]
-        After=docker.socket
-        ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper
-        Requires=docker.socket
-        [Service]
-        Environment="KUBELET_VERSION=${ coreos-hyperkube-tag }"
-        Environment="RKT_OPTS=\
-        --volume=resolv,kind=host,source=/etc/resolv.conf \
-        --mount volume=resolv,target=/etc/resolv.conf"
-        ExecStart=/usr/lib/coreos/kubelet-wrapper \
-          --allow-privileged=true \
-          --api-servers=http://127.0.0.1:8080 \
-          --cloud-provider=aws \
-          --cluster-dns=${ dns-service-ip } \
-          --cluster-domain=cluster.local \
-          --config=/etc/kubernetes/manifests \
-          --register-schedulable=false
-        Restart=always
-        RestartSec=5
-        [Install]
-        WantedBy=multi-user.target
-
   update:
     reboot-strategy: etcd-lock
 EOF
@@ -159,6 +95,6 @@ EOF
     pod-ip-range = "${ var.pod-ip-range }"
     region = "${ var.region }"
     service-ip-range = "${ var.service-ip-range }"
-    ssl-tar = "ssl/k8s-apiserver.tar"
+    ssl-tar = "ssl/k8s-etcd.tar"
   }
 }

modules/etcd/io.tf

@@ -20,5 +20,4 @@ variable "vpc-cidr" {}
 variable "vpc-id" {}
 
 output "depends-id" { value = "${ null_resource.dummy_dependency.id }" }
-output "external-elb" { value = "${ aws_elb.external.dns_name }" }
 output "internal-ips" { value = "${ join(",", aws_instance.etcd.*.public_ip) }" }

modules/master/cloud-config.tf

@@ -0,0 +1,147 @@
+resource "template_file" "cloud-config" {
+  template = <<EOF
+#cloud-config
+
+---
+coreos:
+
+  etcd2:
+    discovery-srv: ${ internal-tld }
+    peer-trusted-ca-file: /etc/kubernetes/ssl/ca.pem
+    peer-client-cert-auth: true
+    peer-cert-file: /etc/kubernetes/ssl/k8s-apiserver.pem
+    peer-key-file: /etc/kubernetes/ssl/k8s-apiserver-key.pem
+    proxy: on
+
+  units:
+    - name: prefetch-hyperkube-container.service
+      command: start
+      content: |
+        [Unit]
+        Description=Accelerate spin up by prefetching hyperkube
+        After=network-online.target
+        [Service]
+        ExecStart=/usr/bin/rkt fetch --trust-keys-from-https \
+          ${ coreos-hyperkube-image }:${ coreos-hyperkube-tag }
+        RemainAfterExit=yes
+        Type=oneshot
+
+    - name: etcd2.service
+      command: start
+
+    - name: flanneld.service
+      command: start
+      drop-ins:
+        - name: 50-network-config.conf
+          content: |
+            [Service]
+            ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config \
+              '{ "Network": "${ pod-ip-range }", "Backend": { "Type": "vxlan" } }'
+            Restart=always
+            RestartSec=10
+
+    - name: docker.service
+      command: start
+      drop-ins:
+        - name: 40-flannel.conf
+          content: |
+            [Unit]
+            After=flanneld.service
+            Requires=flanneld.service
+            [Service]
+            Restart=always
+            RestartSec=10
+        - name: overlay.conf
+          content: |
+            [Service]
+            Environment="DOCKER_OPTS=--storage-driver=overlay"
+
+    - name: s3-get-presigned-url.service
+      command: start
+      content: |
+        [Unit]
+        After=network-online.target
+        Description=Install s3-get-presigned-url
+        Requires=network-online.target
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+        ExecStart=/usr/bin/curl -L -o /opt/bin/s3-get-presigned-url \
+          https://github.com/kz8s/s3-get-presigned-url/releases/download/v0.1/s3-get-presigned-url_linux_amd64
+        ExecStart=/usr/bin/chmod +x /opt/bin/s3-get-presigned-url
+        RemainAfterExit=yes
+        Type=oneshot
+
+    - name: get-ssl.service
+      command: start
+      content: |
+        [Unit]
+        After=s3-get-presigned-url.service
+        Description=Get ssl artifacts from s3 bucket using IAM role
+        Requires=s3-get-presigned-url.service
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /etc/kubernetes/ssl
+        ExecStart=/bin/sh -c "/usr/bin/curl $(/opt/bin/s3-get-presigned-url \
+          ${ region } ${ bucket } ${ ssl-tar }) | tar xv -C /etc/kubernetes/ssl/"
+        RemainAfterExit=yes
+        Type=oneshot
+
+    - name: get-manifests.service
+      command: start
+      content: |
+        [Unit]
+        After=s3-get-presigned-url.service
+        Description=Get kubernetes manifest from s3 bucket using IAM role
+        Requires=s3-get-presigned-url.service
+        [Service]
+        ExecStartPre=-/usr/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStart=/bin/sh -c "/usr/bin/curl $(/opt/bin/s3-get-presigned-url \
+          ${ region } ${ bucket } ${ etc-tar }) | tar xv -C /etc/kubernetes/manifests/"
+        RemainAfterExit=yes
+        Type=oneshot
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        After=docker.socket
+        ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper
+        Requires=docker.socket
+        [Service]
+        Environment="KUBELET_VERSION=${ coreos-hyperkube-tag }"
+        Environment="RKT_OPTS=\
+        --volume=resolv,kind=host,source=/etc/resolv.conf \
+        --mount volume=resolv,target=/etc/resolv.conf"
+        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+          --allow-privileged=true \
+          --api-servers=http://127.0.0.1:8080 \
+          --cloud-provider=aws \
+          --cluster-dns=${ dns-service-ip } \
+          --cluster-domain=cluster.local \
+          --config=/etc/kubernetes/manifests \
+          --register-schedulable=false
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+
+  update:
+    reboot-strategy: etcd-lock
+EOF
+
+  vars {
+    bucket = "${ var.bucket-prefix }"
+    cluster-token = "etcd-cluster-${ var.name }"
+    coreos-hyperkube-image = "${ var.coreos-hyperkube-image }"
+    coreos-hyperkube-tag = "${ var.coreos-hyperkube-tag }"
+    dns-service-ip = "${ var.dns-service-ip }"
+    etc-tar = "/manifests/etc.tar"
+    fqdn = "etcd${ count.index + 1 }.${ var.internal-tld }"
+    hostname = "etcd${ count.index + 1 }"
+    internal-tld = "${ var.internal-tld }"
+    log-group = "k8s-${ var.name }"
+    pod-ip-range = "${ var.pod-ip-range }"
+    region = "${ var.region }"
+    service-ip-range = "${ var.service-ip-range }"
+    ssl-tar = "ssl/k8s-apiserver.tar"
+  }
+}

modules/master/ec2.tf

@@ -0,0 +1,34 @@
+resource "aws_instance" "master" {
+  count = 3
+
+  ami = "${ var.ami-id }"
+  associate_public_ip_address = true
+  iam_instance_profile = "${ var.instance-profile-name }"
+  instance_type = "${ var.instance-type }"
+  key_name = "${ var.key-name }"
+
+  root_block_device {
+    volume_size = 124
+    volume_type = "gp2"
+  }
+
+  source_dest_check = false
+  subnet_id = "${ element( split(",", var.subnet-ids), 0 ) }"
+
+  tags {
+    builtWith = "terraform"
+    Cluster = "${ var.name }"
+    depends-id = "${ var.depends-id }"
+    KubernetesCluster = "${ var.name }" # used by kubelet's aws provider to determine cluster
+    Name = "master${ count.index + 1 }-${ var.name }"
+    role = "apiserver"
+    version = "${ var.coreos-hyperkube-tag}"
+  }
+
+  user_data = "${ template_file.cloud-config.rendered }"
+  vpc_security_group_ids = [ "${ var.master-security-group-id }" ]
+}
+
+resource "null_resource" "dummy_dependency" {
+  depends_on = [ "aws_instance.master" ]
+}

modules/etcd/elb.tf → modules/master/elb.tf

@@ -1,5 +1,5 @@
 resource "aws_elb" "external" {
-  name = "master-ext-k8s-${ var.name }"
+  name = "k8s-master-ext-${replace(var.name, "/(.{0,17})(.*)/", "$1")}"
   subnets = [ "${ split(",", var.subnet-ids) }" ]
   cross_zone_load_balancing = false
   security_groups = [ "${ var.external-elb-security-group-id }" ]
@@ -12,7 +12,7 @@ resource "aws_elb" "external" {
     interval = 30
   }
 
-  instances = [ "${ aws_instance.etcd.*.id }" ]
+  instances = [ "${ aws_instance.master.*.id }" ]
 
   listener {
     instance_port = 443