Define worker nodes with sample workload

bartv2 · bartv2 · commit 9e07b6679d7d · 2023-08-03T22:25:46.000+02:00
diff --git a/Readme.md b/Readme.md
@@ -2,3 +2,14 @@
 - add packages: libvirt-daemon-system qemu-utils qemu-system-x86
 - ssh-keygen -f id_ed25519 -t ed25519 -C terraform@main
 - https://askubuntu.com/a/1293019 AppArmor preventing access to storage pool
+
+run the command below to access the sample workload
+``` bash
+sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl port-forward --address 0.0.0.0 svc/frontend 8080:80
+```
+
+``` bash
+  mkdir -p $HOME/.kube
+  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+  sudo chown $(id -u):$(id -g) $HOME/.kube/config
+```
diff --git a/controlplane.tf b/controlplane.tf
@@ -13,6 +13,7 @@ locals {
   controleplane_ips          = [for i in range(2, 2 + var.controlplane_count) : cidrhost(local.controleplane_network, i)]
   domainname                 = "k8s.lab"
   cluster_endpoint           = "cluster-endpoint.${local.domainname}"
+  cluster_endpoint_ip        = module.control_plane.ip_address[0]
   cluster_endpoint_with_user = "${var.ssh_admin}@${local.cluster_endpoint}"
   kubeadm_token_id           = substr(random_password.kubeadm_token.result, 0, 6)
   kubeadm_token              = join(".", [local.kubeadm_token_id, substr(random_password.kubeadm_token.result, 6, 16)])
@@ -51,18 +52,19 @@ module "control_plane" {
   ]
 
   runcmd = [
-    "install-kubeadm.sh ${local.cluster_endpoint}:6443 ${local.kubeadm_token} ${local.kubeadm_certificate_key} --control-plane --discovery-token-unsafe-skip-ca-verification"
+    "install-kubeadm.sh ${local.cluster_endpoint}:6443 ${local.kubeadm_token} --certificate-key ${local.kubeadm_certificate_key} --control-plane --discovery-token-unsafe-skip-ca-verification"
   ]
 }
 
 resource "ssh_resource" "control_plane_certs" {
-  host        = module.control_plane.ip_address[0]
+  host        = local.cluster_endpoint_ip
   user        = var.ssh_admin
   private_key = var.ssh_private_key
   timeout     = "1m"
 
   triggers = {
     count_changes = length(local.controleplane_ips)
+    workers = var.worker_count
   }
   commands = [
     "sudo kubeadm init phase upload-certs --upload-certs --certificate-key ${local.kubeadm_certificate_key}",
diff --git a/main.tf b/main.tf
@@ -27,3 +27,75 @@ resource "libvirt_network" "default" {
     }
   }
 }
+
+module "workers" {
+  source = "./modules/vm"
+
+  autostart          = false
+  vm_hostname_prefix = "worker"
+  vm_count           = var.worker_count
+  memory             = "1024"
+  vcpu               = 1
+  system_volume      = 10
+
+  time_zone = "CET"
+
+  os_img_url = var.os_img_url
+  pool       = libvirt_pool.cluster.name
+
+  dhcp = true
+#  vm_domain     = local.domainname
+#  ip_address    = local.controleplane_ips
+#  ip_gateway    = cidrhost(local.controleplane_network, 1)
+#  ip_nameserver = cidrhost(local.controleplane_network, 1)
+
+  bridge = libvirt_network.default.bridge
+
+  http_proxy = var.http_proxy
+
+  ssh_admin       = var.ssh_admin
+  ssh_private_key = var.ssh_private_key
+  ssh_keys = [
+    file("${var.ssh_private_key}.pub"),
+  ]
+
+  runcmd = [
+    "install-kubeadm.sh ${local.cluster_endpoint}:6443 ${local.kubeadm_token} --discovery-token-unsafe-skip-ca-verification"
+  ]
+}
+
+resource "ssh_resource" "workers_destroy" {
+  count       = var.worker_count
+  host        = local.cluster_endpoint_ip
+  user        = var.ssh_admin
+  private_key = var.ssh_private_key
+  when        = "destroy"
+  timeout     = "30s"
+
+  commands = [
+    "sudo /usr/local/bin/remove-node.sh ${module.workers.name[count.index]}"
+  ]
+}
+
+resource "ssh_resource" "sample_work" {
+  host        = local.cluster_endpoint_ip
+  user        = var.ssh_admin
+  private_key = var.ssh_private_key
+  timeout     = "11s"
+
+  file {
+    source      = "sample_work.yaml"
+    destination = "/tmp/sample_work.yaml"
+  }
+
+  commands = [
+    "sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f /tmp/sample_work.yaml"
+  ]
+}
+
+output "worker" {
+  value = module.workers
+}
+output "run" {
+  value = ssh_resource.sample_work.result
+}
diff --git a/modules/vm/main.tf b/modules/vm/main.tf
@@ -70,6 +70,10 @@ resource "libvirt_domain" "virt-machine" {
     autoport    = true
   }
 
+  timeouts {
+    create = "10m"
+  }
+
   provisioner "remote-exec" {
     inline = [
       "echo \"Virtual Machine \"$(hostname)\" is UP!\"",
diff --git a/modules/vm/output.tf b/modules/vm/output.tf
@@ -2,5 +2,5 @@ output "name" {
   value = libvirt_domain.virt-machine[*].name
 }
 output "ip_address" {
-  value = libvirt_domain.virt-machine[*].network_interface[0].addresses[0]
+  value = element(libvirt_domain.virt-machine[*].network_interface[0].addresses, 0)
 }
diff --git a/modules/vm/templates/10-containerd-net.conflist b/modules/vm/templates/10-containerd-net.conflist
diff --git a/modules/vm/templates/cloud_init.tpl b/modules/vm/templates/cloud_init.tpl
@@ -30,9 +30,9 @@ packages:
 runcmd:
   - [ systemctl, daemon-reload ]
   - [ systemctl, enable, qemu-guest-agent ]
-  - [ systemctl, start, qemu-guest-agent ]
   - [ systemctl, restart, systemd-networkd ]
 ${runcmd}
+  - [ systemctl, start, qemu-guest-agent ]
 
 fqdn: ${hostname}
 
@@ -105,9 +105,6 @@ write_files:
   - path: /etc/containerd/config.toml
     content: |
         ${ indent(8, file("${path}/templates/containerd-config.toml")) }
-  - path: /etc/cni/net.d/10-containerd-net.conflist
-    content: |
-        ${ indent(8, file("${path}/templates/10-containerd-net.conflist")) }
   - path: /usr/local/bin/install-kubeadm.sh
     permissions: 0o755
     content: |
diff --git a/modules/vm/templates/containerd-config.toml b/modules/vm/templates/containerd-config.toml
@@ -4,7 +4,7 @@ version = 2
   [plugins."io.containerd.grpc.v1.cri"]
     sandbox_image = "registry.k8s.io/pause:3.9"
     [plugins."io.containerd.grpc.v1.cri".cni]
-      bin_dir = "/usr/lib/cni"
+      bin_dir = "/opt/cni/bin"
       conf_dir = "/etc/cni/net.d"
   [plugins."io.containerd.internal.v1.opt"]
     path = "/var/lib/containerd/opt"
diff --git a/modules/vm/templates/install-kubeadm.sh b/modules/vm/templates/install-kubeadm.sh
@@ -12,14 +12,16 @@ sysctl --system
 
 ENDPOINT=$1
 TOKEN=$2
-CERT_KEY=$3
-OTHER_JOIN_ARGS=${*:4}
+CERT_KEY=$4
+OTHER_JOIN_ARGS=${*:3}
 
 INIT=0
 if [ `hostname` = 'controlplane-01' ]; then INIT=1 ; fi
 
 if [ $INIT -eq 1 ]; then
   kubeadm init --control-plane-endpoint=$ENDPOINT --upload-certs --certificate-key $CERT_KEY --token $TOKEN
+  export KUBECONFIG=/etc/kubernetes/admin.conf
+  kubectl apply -f https://github.com/weaveworks/weave/releases/download/v2.8.1/weave-daemonset-k8s.yaml
 else
-  kubeadm join $ENDPOINT --certificate-key $CERT_KEY --token $TOKEN $OTHER_JOIN_ARGS
+  kubeadm join $ENDPOINT --token $TOKEN $OTHER_JOIN_ARGS
 fi
diff --git a/modules/vm/templates/remove-node.sh b/modules/vm/templates/remove-node.sh
@@ -2,13 +2,21 @@
 
 set -e
 
-NODE_NAME=`hostname`
+NODE_NAME=${1:-`hostname`}
 
 export KUBECONFIG=/etc/kubernetes/admin.conf
 
 kubectl drain $NODE_NAME --delete-emptydir-data --force --ignore-daemonsets
 kubectl delete node $NODE_NAME
 
-ETCD_ID=$(etcdctl --endpoints=127.0.0.1:2379 --key /etc/kubernetes/pki/etcd/healthcheck-client.key --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt --cacert /etc/kubernetes/pki/etcd/ca.crt endpoint status  | awk -F, '{print $2}')
+# having param $1 set, implies that it's a worker node
+if [ -v 1 ]; then exit 0 ; fi
 
-etcdctl --endpoints=127.0.0.1:2379 --key /etc/kubernetes/pki/etcd/healthcheck-client.key --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt --cacert /etc/kubernetes/pki/etcd/ca.crt member remove $ETCD_ID
+ETCD_COUNT=$(etcdctl --key /etc/kubernetes/pki/etcd/healthcheck-client.key --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt --cacert /etc/kubernetes/pki/etcd/ca.crt member list  | wc -l)
+
+# check for last controlplane node
+if [ $ETCD_COUNT -eq 1 ]; then exit 0 ; fi
+
+ETCD_ID=$(etcdctl --key /etc/kubernetes/pki/etcd/healthcheck-client.key --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt --cacert /etc/kubernetes/pki/etcd/ca.crt endpoint status  | awk -F, '{print $2}')
+
+etcdctl --key /etc/kubernetes/pki/etcd/healthcheck-client.key --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt --cacert /etc/kubernetes/pki/etcd/ca.crt member remove $ETCD_ID
diff --git a/sample_work.yaml b/sample_work.yaml
@@ -0,0 +1,145 @@
+# from https://kubernetes.io/docs/tutorials/stateless-application/guestbook/
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-leader
+  labels:
+    app: redis
+    role: leader
+    tier: backend
+spec:
+  ports:
+  - port: 6379
+    targetPort: 6379
+  selector:
+    app: redis
+    role: leader
+    tier: backend
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-leader
+  labels:
+    app: redis
+    role: leader
+    tier: backend
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+        role: leader
+        tier: backend
+    spec:
+      containers:
+      - name: leader
+        image: "docker.io/redis:6.0.5"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        ports:
+        - containerPort: 6379
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-follower
+  labels:
+    app: redis
+    role: follower
+    tier: backend
+spec:
+  ports:
+    # the port that this service should serve on
+  - port: 6379
+  selector:
+    app: redis
+    role: follower
+    tier: backend
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-follower
+  labels:
+    app: redis
+    role: follower
+    tier: backend
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+        role: follower
+        tier: backend
+    spec:
+      containers:
+      - name: follower
+        image: gcr.io/google_samples/gb-redis-follower:v2
+        env:
+        - name: GET_HOSTS_FROM
+          value: "dns"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        ports:
+        - containerPort: 6379
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend
+  labels:
+    app: guestbook
+    tier: frontend
+spec:
+  # if your cluster supports it, uncomment the following to automatically create
+  # an external load-balanced IP for the frontend service.
+  # type: LoadBalancer
+  #type: LoadBalancer
+  ports:
+    # the port that this service should serve on
+  - port: 80
+  selector:
+    app: guestbook
+    tier: frontend
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+        app: guestbook
+        tier: frontend
+  template:
+    metadata:
+      labels:
+        app: guestbook
+        tier: frontend
+    spec:
+      containers:
+      - name: php-redis
+        image: gcr.io/google_samples/gb-frontend:v5
+        env:
+        - name: GET_HOSTS_FROM
+          value: "dns"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        ports:
+        - containerPort: 80
diff --git a/variables.tf b/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -2,5 +2,5 @@ output "name" {`
`2`	`2`	`value = libvirt_domain.virt-machine[*].name`
`3`	`3`	`}`
`4`	`4`	`output "ip_address" {`
`5`		`- value = libvirt_domain.virt-machine[*].network_interface[0].addresses[0]`
	`5`	`+ value = element(libvirt_domain.virt-machine[*].network_interface[0].addresses, 0)`
`6`	`6`	`}`