Merge branch 'master' into log-driver-variable

Author: webyneter

Dockerfile

@@ -11,15 +11,15 @@ RUN apt-get update \
   && apt-get clean
 
 COPY tools /usr/local/bin
-RUN curl -sL "https://releases.hashicorp.com/terraform/0.7.2/terraform_0.7.2_linux_amd64.zip"> terraform.zip \
+RUN curl -sL "https://releases.hashicorp.com/terraform/0.9.11/terraform_0.9.11_linux_amd64.zip"> terraform.zip \
   && unzip terraform.zip \
   && mv terraform /usr/local/bin
 
-RUN curl -sL "https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip" > packer.zip \
+RUN curl -sL "https://releases.hashicorp.com/packer/1.0.3/packer_1.0.3_linux_amd64.zip" > packer.zip \
   && unzip packer.zip \
   && mv packer /usr/local/bin
 
-RUN curl -sL -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64 && chmod +x /usr/local/bin/dumb-init
+RUN curl -sL -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 && chmod +x /usr/local/bin/dumb-init
 
 ENTRYPOINT ["/usr/local/bin/dumb-init"]
 

Makefile

@@ -33,8 +33,13 @@ endif
 
 install-tools: $(tools)
 
+ifeq (${platform},Darwin)
 /usr/local/bin/%: ./tools/%
 	install -S -m 0755 $< /usr/local/bin
+else
+/usr/local/bin/%: ./tools/%
+	install -m 0755 $< /usr/local/bin
+endif
 
 amis:
 	pack-ami build -p ./packer -t base -r

Readme.md

@@ -155,12 +155,16 @@ traffic in and out of the different subnets. The Stack terraform will automatica
 
 Traffic from each internal subnet to the outside world will run through the associated NAT gateway.
 
+Alternatively, setting the `use_nat_instances` VPC module variable to true, will use [EC2 NAT instances][nat-instances] instead of the NAT gateway. NAT instances cost less than the NAT gateway, can be shutdown when not in use, and may be preferred in development environments. By default, NAT instances will not use [Elastic IPs][elastic-ip] to avoid a small hourly charge if the NAT instances are not running full time. To use Elastic IPs for the NAT instances, set the `use_eip_with_nat_instances` VPC module variable to true.
+
 For further reading, check out these sources:
 
 - [Recommended Address Space](http://serverfault.com/questions/630022/what-is-the-recommended-cidr-when-creating-vpc-on-aws)
 - [Practical VPC Design](https://medium.com/aws-activate-startup-blog/practical-vpc-design-8412e1a18dcc)
 
 [nat-gateway]: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-nat-gateway.html
+[nat-instances]: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_NAT_Instance.html
+[elastic-ip]: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html
 
 ### Instances
 
@@ -194,7 +198,7 @@ For more complicated service discovery which handles cases like versioning, we'd
 
 ### Bastion
 
-The bastion host acts as the "jump point" for the rest of the infrastructure. Since most of our instances aren't exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
+The bastion host acts as the "jump point" for the rest of the infrastructure. Since most of our instances are not exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
 
 The bastion is provisioned using the key name that you pass to the stack (and hopefully have stored somewhere). If you ever need to access an instance directly, you can do it by "jumping through" the bastion:
 

defaults/main.tf

@@ -23,6 +23,8 @@ variable "cidr" {
 }
 
 variable "default_ecs_ami" {
+  type = "map"
+
   default = {
     us-east-1      = "ami-dde4e6ca"
     us-west-1      = "ami-6d21770d"
@@ -39,6 +41,8 @@ variable "default_ecs_ami" {
 
 # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html#attach-bucket-policy
 variable "default_log_account_ids" {
+  type = "map"
+
   default = {
     us-east-1      = "127311923021"
     us-west-2      = "797873946194"

docs.md

@@ -44,6 +44,10 @@ Usage:
 | cidr | the CIDR block to provision for the VPC, if set to something other than the default, both internal_subnets and external_subnets have to be defined as well | `10.30.0.0/16` | no |
 | internal_subnets | a list of CIDRs for internal subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones | `<list>` | no |
 | external_subnets | a list of CIDRs for external subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones | `<list>` | no |
+| use_nat_instances | use NAT EC2 instances instead of the NAT gateway service | `false` | no |
+| use_eip_with_nat_instances | use Elastic IPs with NAT instances if `use_nat_instances` is true | `false` | no |
+| nat_instance_type | the EC2 instance type for NAT instances if `use_nat_instances` is true | `t2.nano` | no |
+| nat_instance_ssh_key_name | the name of the ssh key to use with NAT instances if `use_nat_instances` is true | "" | no |
 | availability_zones | a comma-separated list of availability zones, defaults to all AZ of the region, if set to something other than the defaults, both internal_subnets and external_subnets have to be defined as well | `<list>` | no |
 | bastion_instance_type | Instance type for the bastion | `t2.micro` | no |
 | ecs_cluster_name | the name of the cluster, if not specified the variable name will be used | `` | no |
@@ -88,7 +92,7 @@ Usage:
 # bastion
 
 The bastion host acts as the "jump point" for the rest of the infrastructure.
-Since most of our instances aren't exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
+Since most of our instances are not exposed to the external internet, the bastion acts as the gatekeeper for any direct SSH access.
 The bastion is provisioned using the key name that you pass to the stack (and hopefully have stored somewhere).
 If you ever need to access an instance directly, you can do it by "jumping through" the bastion.
 
@@ -130,7 +134,7 @@ Usage:
 # defaults
 
 This module is used to set configuration defaults for the AWS infrastructure.
-It doesn't provide much value when used on its own because terraform makes it
+It does not provide much value when used on its own because terraform makes it
 hard to do dynamic generations of things like subnets, for now it's used as
 a helper module for the stack.
 
@@ -248,7 +252,7 @@ Usage:
 | instance_type | The instance type to use, e.g t2.small | - | yes |
 | instance_ebs_optimized | When set to true the instance will be launched with EBS optimized turned on | `true` | no |
 | min_size | Minimum instance count | `3` | no |
-| max_size | Maxmimum instance count | `100` | no |
+| max_size | Maximum instance count | `100` | no |
 | desired_capacity | Desired instance count | `3` | no |
 | associate_public_ip_address | Should created instances be publicly accessible (if the SG allows) | `false` | no |
 | root_volume_size | Root volume size in GB | `25` | no |

ecs-cluster/main.tf

@@ -93,7 +93,7 @@ variable "desired_capacity" {
 
 variable "associate_public_ip_address" {
   description = "Should created instances be publicly accessible (if the SG allows)"
-  default = false
+  default     = false
 }
 
 variable "root_volume_size" {
@@ -126,35 +126,6 @@ variable "extra_cloud_config_content" {
   default     = ""
 }
 
-resource "aws_security_group" "cluster" {
-  name        = "${var.name}-ecs-cluster"
-  vpc_id      = "${var.vpc_id}"
-  description = "Allows traffic from and to the EC2 instances of the ${var.name} ECS cluster"
-
-  ingress {
-    from_port       = 0
-    to_port         = 0
-    protocol        = -1
-    security_groups = ["${split(",", var.security_groups)}"]
-  }
-
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = -1
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
-  tags {
-    Name        = "ECS cluster (${var.name})"
-    Environment = "${var.environment}"
-  }
-
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
 resource "aws_ecs_cluster" "main" {
   name = "${var.name}"
 
@@ -198,7 +169,7 @@ resource "aws_launch_configuration" "main" {
   ebs_optimized               = "${var.instance_ebs_optimized}"
   iam_instance_profile        = "${var.iam_instance_profile}"
   key_name                    = "${var.key_name}"
-  security_groups             = ["${aws_security_group.cluster.id}"]
+  security_groups             = ["${split(",", var.security_groups)}"]
   user_data                   = "${data.template_cloudinit_config.cloud_config.rendered}"
   associate_public_ip_address = "${var.associate_public_ip_address}"
 
@@ -382,8 +353,3 @@ resource "aws_cloudwatch_metric_alarm" "memory_low" {
 output "name" {
   value = "${var.name}"
 }
-
-// The cluster security group ID.
-output "security_group_id" {
-  value = "${aws_security_group.cluster.id}"
-}

iam-role/main.tf

@@ -95,9 +95,9 @@ EOF
 }
 
 resource "aws_iam_instance_profile" "default_ecs" {
-  name  = "ecs-instance-profile-${var.name}-${var.environment}"
-  path  = "/"
-  roles = ["${aws_iam_role.default_ecs_role.name}"]
+  name = "ecs-instance-profile-${var.name}-${var.environment}"
+  path = "/"
+  role = "${aws_iam_role.default_ecs_role.name}"
 }
 
 output "default_ecs_role_id" {

main.tf

@@ -46,28 +46,31 @@ variable "cidr" {
 }
 
 variable "internal_subnets" {
+  type        = "list"
   description = "a list of CIDRs for internal subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones"
-  default     = ["10.30.0.0/19" ,"10.30.64.0/19", "10.30.128.0/19"]
+  default     = ["10.30.0.0/19", "10.30.64.0/19", "10.30.128.0/19"]
 }
 
 variable "external_subnets" {
+  type        = "list"
   description = "a list of CIDRs for external subnets in your VPC, must be set if the cidr variable is defined, needs to have as many elements as there are availability zones"
   default     = ["10.30.32.0/20", "10.30.96.0/20", "10.30.160.0/20"]
 }
 
 variable "availability_zones" {
+  type        = "list"
   description = "a comma-separated list of availability zones, defaults to all AZ of the region, if set to something other than the defaults, both internal_subnets and external_subnets have to be defined as well"
   default     = ["us-west-2a", "us-west-2b", "us-west-2c"]
 }
 
 variable "bastion_instance_type" {
   description = "Instance type for the bastion"
-  default = "t2.micro"
+  default     = "t2.micro"
 }
 
 variable "ecs_cluster_name" {
   description = "the name of the cluster, if not specified the variable name will be used"
-  default = ""
+  default     = ""
 }
 
 variable "ecs_instance_type" {
@@ -124,6 +127,11 @@ variable "ecs_security_groups" {
   default     = ""
 }
 
+variable "ecs_extra_security_groups" {
+  description = "A comma separated list of security groups added to the default security groups of the stack"
+  default     = ""
+}
+
 variable "ecs_ami" {
   description = "The AMI that will be used to launch EC2 instances in the ECS cluster"
   default     = ""
@@ -202,35 +210,34 @@ module "iam_role" {
 }
 
 module "ecs_cluster" {
-  source                 = "./ecs-cluster"
-  name                   = "${coalesce(var.ecs_cluster_name, var.name)}"
-  environment            = "${var.environment}"
-  vpc_id                 = "${module.vpc.id}"
-  image_id               = "${coalesce(var.ecs_ami, module.defaults.ecs_ami)}"
-  subnet_ids             = "${module.vpc.internal_subnets}"
-  key_name               = "${var.key_name}"
-  instance_type          = "${var.ecs_instance_type}"
-  instance_ebs_optimized = "${var.ecs_instance_ebs_optimized}"
-  iam_instance_profile   = "${module.iam_role.profile}"
-  min_size               = "${var.ecs_min_size}"
-  max_size               = "${var.ecs_max_size}"
-  desired_capacity       = "${var.ecs_desired_capacity}"
-  region                 = "${var.region}"
-  availability_zones     = "${module.vpc.availability_zones}"
-  root_volume_size       = "${var.ecs_root_volume_size}"
-  docker_volume_size     = "${var.ecs_docker_volume_size}"
-  docker_auth_type       = "${var.ecs_docker_auth_type}"
-  docker_auth_data       = "${var.ecs_docker_auth_data}"
-  security_groups        = "${coalesce(var.ecs_security_groups, format("%s,%s,%s", module.security_groups.internal_ssh, module.security_groups.internal_elb, module.security_groups.external_elb))}"
-  extra_cloud_config_type     = "${var.extra_cloud_config_type}"
-  extra_cloud_config_content  = "${var.extra_cloud_config_content}"
+  source                     = "./ecs-cluster"
+  name                       = "${coalesce(var.ecs_cluster_name, var.name)}"
+  environment                = "${var.environment}"
+  vpc_id                     = "${module.vpc.id}"
+  image_id                   = "${coalesce(var.ecs_ami, module.defaults.ecs_ami)}"
+  subnet_ids                 = "${module.vpc.internal_subnets}"
+  key_name                   = "${var.key_name}"
+  instance_type              = "${var.ecs_instance_type}"
+  instance_ebs_optimized     = "${var.ecs_instance_ebs_optimized}"
+  iam_instance_profile       = "${module.iam_role.profile}"
+  min_size                   = "${var.ecs_min_size}"
+  max_size                   = "${var.ecs_max_size}"
+  desired_capacity           = "${var.ecs_desired_capacity}"
+  region                     = "${var.region}"
+  availability_zones         = "${module.vpc.availability_zones}"
+  root_volume_size           = "${var.ecs_root_volume_size}"
+  docker_volume_size         = "${var.ecs_docker_volume_size}"
+  docker_auth_type           = "${var.ecs_docker_auth_type}"
+  docker_auth_data           = "${var.ecs_docker_auth_data}"
+  security_groups            = "${coalesce(var.ecs_security_groups, format("%s,%s,%s", module.security_groups.internal_ssh, module.security_groups.internal_elb, module.security_groups.external_elb))}"
+  extra_cloud_config_type    = "${var.extra_cloud_config_type}"
+  extra_cloud_config_content = "${var.extra_cloud_config_content}"
 }
 
 module "s3_logs" {
   source                  = "./s3-logs"
   name                    = "${var.name}"
   environment             = "${var.environment}"
-  account_id              = "${module.defaults.s3_logs_account_id}"
   logs_expiration_enabled = "${var.logs_expiration_enabled}"
   logs_expiration_days    = "${var.logs_expiration_days}"
 }
@@ -329,3 +336,13 @@ output "internal_route_tables" {
 output "external_route_tables" {
   value = "${module.vpc.external_rtb_id}"
 }
+
+// The external ssh security group ID.
+output "external_ssh" {
+  value = "${module.security_groups.external_ssh}"
+}
+
+// The internal ssh security group ID.
+output "internal_ssh" {
+  value = "${module.security_groups.internal_ssh}"
+}

packer/base/packer.yml

@@ -1,7 +1,26 @@
 ---
+# Latest xenial	16.04 LTS	amd64	hvm:ebs-ssd Releases 
+#  us-gov-west-1	20170619.1	ami-939412f2
+#  us-east-2	    20170619.1	ami-8b92b4ee
+#  sa-east-1	    20170619.1	ami-34afc458
+#  eu-central-1	    20170619.1	ami-1c45e273
+#  us-west-1	    20170619.1	ami-73f7da13
+#  us-west-2	    20170619.1	ami-835b4efa
+#  ap-northeast-2	20170619.1	ami-94d20dfa
+#  ca-central-1	    20170619.1	ami-7ed56a1a
+#  eu-west-2	    20170619.1	ami-cc7066a8
+#  ap-southeast-1	20170619.1	ami-2378f540
+#  eu-west-1	    20170619.1	ami-6d48500b
+#  ap-southeast-2	20170619.1	ami-e94e5e8a
+#  ap-northeast-1	20170619.1	ami-785c491f
+#  us-east-1	    20170619.1	ami-d15a75c7
+#  ap-south-1	    20170619.1	ami-49e59a26
+#  cn-north-1	    20170303	ami-a163b4cc
+
+
 # https://www.packer.io/docs/builders/amazon-ebs.html
 ami:
-  source_ami: ami-e6d5d2f1
+  source_ami: ami-d15a75c7
   region: us-east-1
   instance_type: c4.2xlarge
   ssh_username: ubuntu

packer/base/root/etc/apt/preferences.d/docker-engine.pref

@@ -0,0 +1,3 @@
+Package: *
+Pin: release o=Docker
+Pin-Priority: 900

packer/base/scripts/base.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 set -e
 
+export DEBIAN_FRONTEND=noninteractive
 systemctl disable apt-daily.service
 systemctl disable apt-daily.timer
 
 apt-get update -y
-apt-get upgrade -y
 
 apt-get install -y \
         build-essential  \
@@ -35,8 +35,12 @@ apt-get install -y \
         ntp \
         logrotate \
         dhcping \
+        nfs-common \
+        curl \
+        unzip \
+        jq \
         dhcpdump
 
 pip install awscli
 
-apt-get dist-upgrade -y
+apt-get upgrade -y

packer/base/scripts/docker.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+export DEBIAN_FRONTEND=noninteractive
+
 apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
 echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' > /etc/apt/sources.list.d/docker.list
 
@@ -9,7 +11,6 @@ apt-get purge -y lxc-docker
 apt-cache policy docker-engine
 
 apt-get install -o Dpkg::Options::="--force-confold" -y \
-        linux-image-extra-$(uname -r) \
         docker-engine
 
 gpasswd -a ubuntu docker