Refresh the builder (#580)

* updates builder to use common stuff like email and monitoring
* Don't hardcode instance names for singletons (probably problematic if/when we restart
* Latest images

Makefile

@@ -20,7 +20,7 @@ config.json: make_json.py | $(PYTHON)
 
 .PHONY: packer
 packer: config.json ## Builds the base image for compiler explorer nodes
-	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer.json
+	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-node.json
 
 .PHONY: packer-local
 packer-local: config.json ## Builds a local docker version of the compiler explorer node image

bin/lib/amazon.py

@@ -51,6 +51,7 @@ def _create_anon_s3_client():
 
 
 ec2 = LazyObjectWrapper(lambda: boto3.resource('ec2'))
+ec2_client = LazyObjectWrapper(lambda: boto3.client('ec2'))
 s3 = LazyObjectWrapper(lambda: boto3.resource('s3'))
 as_client = LazyObjectWrapper(lambda: boto3.client('autoscaling'))
 elb_client = LazyObjectWrapper(lambda: boto3.client('elbv2'))

bin/lib/instance.py

@@ -1,12 +1,32 @@
+import functools
 import logging
 import subprocess
 from typing import Dict, Optional
 
-from lib.amazon import ec2, as_client, elb_client, get_releases, release_for
+from lib.amazon import ec2, ec2_client, as_client, elb_client, get_releases, release_for
 from lib.ssh import exec_remote, can_ssh_to
 
 STATUS_FORMAT = '{: <16} {: <20} {: <10} {: <12} {: <11} {: <11} {: <14}'
-logger = logging.getLogger('ssh')
+logger = logging.getLogger('instance')
+
+
+@functools.lru_cache()
+def _singleton_instance(name: str):
+    result = ec2_client.describe_instances(Filters=[
+        {'Name': 'tag:Name', 'Values': [name]},
+        {'Name': 'instance-state-name', 'Values': ['stopped', 'stopping', 'running', 'pending']},
+    ])
+    reservations = result['Reservations']
+    if len(reservations) == 0:
+        raise RuntimeError(f"No instance named '{name}' found")
+    if len(reservations) > 1:
+        raise RuntimeError(f"Multiple instances named '{name}' found ({reservations}")
+    instances = reservations[0]['Instances']
+    if len(instances) == 0:
+        raise RuntimeError(f"No instance named '{name}' found")
+    if len(instances) > 1:
+        raise RuntimeError(f"Multiple instances named '{name}' found ({instances}")
+    return ec2.Instance(id=instances[0]['InstanceId'])
 
 
 class Instance:
@@ -70,7 +90,7 @@ def address(self):
 
     @staticmethod
     def instance():
-        return AdminInstance(ec2.Instance(id='i-0988cd194a4a8a2c0'))
+        return AdminInstance(_singleton_instance("AdminNode"))
 
 
 class ConanInstance:
@@ -82,7 +102,7 @@ def __init__(self, instance):
 
     @staticmethod
     def instance():
-        return ConanInstance(ec2.Instance(id='i-0a3f201b6378db03f'))
+        return ConanInstance(_singleton_instance("ConanNode"))
 
 
 class BuilderInstance:
@@ -94,7 +114,7 @@ def __init__(self, instance):
 
     @staticmethod
     def instance():
-        return BuilderInstance(ec2.Instance(id='i-06e4bfd4430a8955a'))
+        return BuilderInstance(_singleton_instance('Builder'))
 
     def start(self):
         self.instance.start()

crontab.builder

@@ -0,0 +1,4 @@
+MAILTO=compiler-explorer-admin@googlegroups.com
+PATH=/home/ubuntu/infra/.env/bin:/home/ubuntu/infra/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# m h  dom mon dow   command

mount-all-img.sh

@@ -17,4 +17,6 @@ for img_file in "${IMG_DIR}"/**/*.img; do
     fi
 done
 
+echo -n "Waiting for mounts to complete..."
 wait
+echo "done"

packer-admin.json

@@ -51,9 +51,9 @@
     { "type": "file", "source": "packer", "destination": "/home/ubuntu/" },
     {
       "type": "shell",
-      "execute_command" : "{{ .Vars }} sudo -E sh '{{ .Path }}'",
+      "execute_command" : "{{ .Vars }} sudo -E bash '{{ .Path }}'",
       "inline": [
-        "set -e",
+        "set -euo pipefail",
         "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
         "export DEBIAN_FRONTEND=noninteractive",
         "cp /home/ubuntu/packer/known_hosts /home/ubuntu/.ssh/",

packer-builder.json

@@ -43,6 +43,7 @@
   ],
 
   "variables": {
+    "BRANCH": "main",
     "MY_ACCESS_KEY": "",
     "MY_SECRET_KEY": ""
   },
@@ -51,18 +52,18 @@
     { "type": "file", "source": "packer", "destination": "/home/ubuntu/" },
     {
       "type": "shell",
-      "execute_command" : "{{ .Vars }} sudo -E sh '{{ .Path }}'",
+      "execute_command" : "{{ .Vars }} sudo -E bash '{{ .Path }}'",
       "inline": [
-        "set -e",
+        "set -euo pipefail",
         "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
         "export DEBIAN_FRONTEND=noninteractive",
         "cp /home/ubuntu/packer/known_hosts /home/ubuntu/.ssh/",
         "rm -rf /home/ubuntu/packer",
         "apt-get -y update",
         "apt-get -y install git",
-        "git clone https://github.com/compiler-explorer/infra.git /home/ubuntu/infra",
-        "chown -R ubuntu:ubuntu /home/ubuntu/infra",
-        "/home/ubuntu/infra/setup-builder.sh"
+        "git clone -b {{user `BRANCH`}} https://github.com/compiler-explorer/infra.git /infra",
+        "cd /infra",
+        "env PACKER_SETUP=yes bash setup-builder.sh 2>&1 | tee /tmp/setup.log"
       ]
     }
   ]

packer-conan.json

@@ -51,9 +51,9 @@
     { "type": "file", "source": "packer", "destination": "/home/ubuntu/" },
     {
       "type": "shell",
-      "execute_command" : "{{ .Vars }} sudo -E sh '{{ .Path }}'",
+      "execute_command" : "{{ .Vars }} sudo -E bash '{{ .Path }}'",
       "inline": [
-        "set -e",
+        "set -euo pipefail",
         "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
         "export DEBIAN_FRONTEND=noninteractive",
         "cp /home/ubuntu/packer/known_hosts /home/ubuntu/.ssh/",

packer.json → packer-node.json

@@ -52,9 +52,9 @@
     { "type": "file", "source": "packer", "destination": "/home/ubuntu/" },
     {
       "type": "shell",
-      "execute_command" : "{{ .Vars }} sudo -E sh '{{ .Path }}'",
+      "execute_command" : "{{ .Vars }} sudo -E bash '{{ .Path }}'",
       "inline": [
-        "set -e",
+        "set -euo pipefail",
         "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
         "export DEBIAN_FRONTEND=noninteractive",
         "mkdir -p /root/.ssh",
@@ -65,7 +65,7 @@
         "apt-get -y install git",
         "git clone -b {{user `BRANCH`}} https://github.com/compiler-explorer/infra.git /infra",
         "cd /infra",
-        "env PACKER_SETUP=yes bash setup.sh 2>&1 | tee /tmp/setup.log"
+        "env PACKER_SETUP=yes bash setup-node.sh 2>&1 | tee /tmp/setup.log"
       ]
     }
   ]

setup-admin.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -ex
+set -exuo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "${DIR}"
@@ -14,7 +14,7 @@ fi
 
 env EXTRA_NFS_ARGS="" "${DIR}/setup-common.sh"
 
-apt -y install mosh fish jq ssmtp cronic subversion upx gdb autojump zlib1g-dev m4 python3 python3-venv python3.8 python3.8-venv libc6-dev-i386
+apt -y install mosh fish jq cronic subversion upx gdb autojump zlib1g-dev m4 python3 python3-venv python3.8 python3.8-venv libc6-dev-i386
 chsh ubuntu -s /usr/bin/fish
 
 cd /home/ubuntu/infra
@@ -33,28 +33,10 @@ chown -R ubuntu:ubuntu /home/ubuntu/infra
 sudo -u ubuntu fish setup.fish
 crontab -u ubuntu crontab.admin
 
-# Configure email
-SMTP_PASS=$(aws ssm get-parameter --name /admin/smtp_pass | jq -r .Parameter.Value)
-cat >/etc/ssmtp/ssmtp.conf <<EOF
-root=postmaster
-mailhub=email-smtp.us-east-1.amazonaws.com
-hostname=compiler-explorer.com
-FromLineOverride=NO
-AuthUser=AKIAJZWPG4D3SSK45LJA
-AuthPass=${SMTP_PASS}
-UseTLS=YES
-UseSTARTTLS=YES
-EOF
-cat >/etc/ssmtp/revaliases <<EOF
-ubuntu:[email protected]:email-smtp.us-east-1.amazonaws.com
-EOF
-
-chfn -f 'Compiler Explorer Admin' ubuntu
-chmod 640 /etc/ssmtp/*
-
 echo admin-node >/etc/hostname
 hostname admin-node
 sed -i "/127.0.0.1/c 127.0.0.1 localhost admin-node" /etc/hosts
+sed -i "/preserve_hostname/c preserve_hostname: true" /etc/cloud/cloud.cfg
 
 if ! grep vm.min_free_kbytes /etc/sysctl.conf; then
   echo "vm.min_free_kbytes=65536" >>/etc/sysctl.conf

setup-builder-startup.sh

@@ -6,14 +6,11 @@ set -euo pipefail
 
 systemctl stop docker
 umount /ephemeral || true
-sfdisk -f /dev/nvme1n1 <<EOF
-label: dos
-label-id: 0x7d7e658b
-device: /dev/nvme1n1
-unit: sectors
-
-/dev/nvme1n1p1 : start=     3999744, size=   120999936, type=82
-/dev/nvme1n1p2 : start=   124999680, size=   656248832, type=83
+# swap was 57GB (!) hopefully we really don't need that much
+SWAP_SIZE=$((8 * 1024 * 1024 * 1024 / 512))
+sfdisk /dev/nvme1n1 <<EOF
+,${SWAP_SIZE},82
+;
 EOF
 sync
 sleep 2 # let the device get registered

setup-builder.sh

@@ -1,58 +1,32 @@
 #!/bin/bash
 
-set -ex
+set -exuo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-if [[ "$1" != "--updated" ]]; then
-    sudo -u ubuntu git -C "${DIR}" pull
-    pwd
-    exec bash "${BASH_SOURCE[0]}" --updated
-    exit 0
-fi
-
-if [[ ! -f /updated.2 ]]; then
-    wget -qO- https://get.docker.com/ | sh
-    touch /updated.2
-fi
-
 env EXTRA_NFS_ARGS="" "${DIR}/setup-common.sh"
 
-apt -y install python2.7 mosh fish jq ssmtp cronic subversion upx gdb
-chsh ubuntu -s /usr/bin/fish
+wget -qO- https://get.docker.com/ | sh
+usermod -aG docker ubuntu
 
-cd /home/ubuntu/infra
-pip install --upgrade -r requirements.txt
+apt -y install mosh fish cronic subversion upx gdb
+chsh ubuntu -s /usr/bin/fish
 
-# Install private and public keys
 aws ssm get-parameter --name /admin/ce_private_key | jq -r .Parameter.Value >/home/ubuntu/.ssh/id_rsa
-
 chmod 600 /home/ubuntu/.ssh/id_rsa
 aws s3 cp s3://compiler-explorer/authorized_keys/admin.key /home/ubuntu/.ssh/id_rsa.pub
 chown -R ubuntu:ubuntu /home/ubuntu/.ssh
-chown -R ubuntu:ubuntu /home/ubuntu/infra
 
 sudo -u ubuntu fish setup.fish
+crontab -u ubuntu crontab.builder
+
+echo builder >/etc/hostname
+hostname builder
+sed -i "/127.0.0.1/c 127.0.0.1 localhost builder" /etc/hosts
+sed -i "/preserve_hostname/c preserve_hostname: true" /etc/cloud/cloud.cfg
+
+mv /infra /home/ubuntu/infra
+chown -R ubuntu:ubuntu /home/ubuntu/infra
+sudo -u ubuntu make -C /home/ubuntu/infra ce
 
-# Configure email
-SMTP_PASS=$(aws ssm get-parameter --name /admin/smtp_pass | jq -r .Parameter.Value)
-cat >/etc/ssmtp/ssmtp.conf <<EOF
-root=postmaster
-mailhub=email-smtp.us-east-1.amazonaws.com
-hostname=compiler-explorer.com
-FromLineOverride=NO
-AuthUser=AKIAJZWPG4D3SSK45LJA
-AuthPass=${SMTP_PASS}
-UseTLS=YES
-UseSTARTTLS=YES
-EOF
-cat >/etc/ssmtp/revaliases <<EOF
-ubuntu:[email protected]:email-smtp.us-east-1.amazonaws.com
-EOF
-
-chfn -f 'Compiler Explorer Admin' ubuntu
-chmod 640 /etc/ssmtp/*
-
-echo builder-node > /etc/hostname
-hostname builder-node
-sed -i "/127.0.0.1/c 127.0.0.1 localhost builder-node" /etc/hosts
+ln -s /efs/squash-images /opt/squash-images

setup-common.sh

@@ -1,15 +1,13 @@
 #!/bin/bash
 
-set -ex
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+set -exuo pipefail
 
 # https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running
 wait_for_apt() {
-    while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
-        echo "Waiting for other software managers to finish..."
-        sleep 5
-    done
+  while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
+    echo "Waiting for other software managers to finish..."
+    sleep 5
+  done
 }
 
 # Sometimes it seems auto apt takes a while to kick in...
@@ -20,19 +18,31 @@ wait_for_apt
 
 apt-get -y update
 apt-get -y upgrade --force-yes
-apt-get -y install unzip libwww-perl libdatetime-perl nfs-common jq python3-pip wget cachefilesd qemu-user-static libc6-arm64-cross libtinfo5
+apt-get -y install \
+  jq \
+  libc6-arm64-cross \
+  libdatetime-perl \
+  libtinfo5 \
+  libwww-perl \
+  nfs-common \
+  python3-pip \
+  python3-venv \
+  qemu-user-static \
+  ssmtp \
+  unzip \
+  wget
+
 apt-get -y autoremove
 pip3 install --upgrade pip
 hash -r pip
 pip install --upgrade awscli
-touch /updated
 
 mkdir -p /root/.aws /home/ubuntu/.aws
 echo -e "[default]\nregion=us-east-1" | tee /root/.aws/config /home/ubuntu/.aws/config
 chown -R ubuntu /home/ubuntu/.aws
 
 get_conf() {
-    aws ssm get-parameter --name $1 | jq -r .Parameter.Value
+  aws ssm get-parameter --name "$1" | jq -r .Parameter.Value
 }
 
 LOG_DEST_HOST=$(get_conf /compiler-explorer/logDestHost)
@@ -97,9 +107,28 @@ chmod 600 /etc/grafana/agent.yaml.tpl
 
 mkdir -p /efs
 if ! grep "/efs nfs" /etc/fstab; then
-    echo "fs-db4c8192.efs.us-east-1.amazonaws.com:/ /efs nfs nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport${EXTRA_NFS_ARGS} 0 0" >>/etc/fstab
+  echo "fs-db4c8192.efs.us-east-1.amazonaws.com:/ /efs nfs nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport${EXTRA_NFS_ARGS} 0 0" >>/etc/fstab
 fi
 
+# Configure email
+SMTP_PASS=$(aws ssm get-parameter --name /admin/smtp_pass | jq -r .Parameter.Value)
+cat >/etc/ssmtp/ssmtp.conf <<EOF
+root=postmaster
+mailhub=email-smtp.us-east-1.amazonaws.com
+hostname=compiler-explorer.com
+FromLineOverride=NO
+AuthUser=AKIAJZWPG4D3SSK45LJA
+AuthPass=${SMTP_PASS}
+UseTLS=YES
+UseSTARTTLS=YES
+EOF
+cat >/etc/ssmtp/revaliases <<EOF
+ubuntu:[email protected]:email-smtp.us-east-1.amazonaws.com
+EOF
+
+chfn -f 'Compiler Explorer Admin' ubuntu
+chmod 640 /etc/ssmtp/*
+
 mount -a
 
 cd /home/ubuntu/