Merge remote-tracking branch 'origin/rootless' into CR-17416-rootless

danielm-codefresh · danielm-codefresh · commit 33d83a818bf0 · 2023-05-25T09:57:19.000+03:00
diff --git a/Dockerfile b/Dockerfile
@@ -11,22 +11,28 @@ RUN go mod tidy
 COPY cleaner/dind-cleaner/cmd ./cmd/
 
 RUN CGO_ENABLED=0 go build -o /usr/local/bin/dind-cleaner ./cmd && \
-    chmod +x /usr/local/bin/dind-cleaner && \
-    rm -rf /go/*
+  chmod +x /usr/local/bin/dind-cleaner && \
+  rm -rf /go/*
 
 # bolter
 FROM golang:1.19-alpine3.16 AS bolter
+RUN apk add git
 RUN go install github.com/hasit/bolter@v0.0.0-20210331045447-e1283cecdb7b
 
 # node-exporter
 FROM quay.io/prometheus/node-exporter:v1.5.0 AS node-exporter
 
 # Main
-FROM docker:${DOCKER_VERSION}-dind
+FROM docker:${DOCKER_VERSION}-dind-rootless
 
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.11/main' >> /etc/apk/repositories \
+USER root
+
+RUN chown -R $(id -u rootless) /var /run /lib /home /etc/ssl /etc/apk
+
+# Add community for fuse-overlayfs
+RUN echo -en "https://dl-cdn.alpinelinux.org/alpine/v$(cut -d'.' -f1,2 /etc/alpine-release)/main\nhttps://dl-cdn.alpinelinux.org/alpine/v$(cut -d'.' -f1,2 /etc/alpine-release)/community" > /etc/apk/repositories \
   && apk upgrade \
-  && apk add bash jq --no-cache \
+  && apk add bash jq fuse-overlayfs --no-cache \
   && rm -rf /var/cache/apk/*
 
 COPY --from=node-exporter /bin/node_exporter /bin/
@@ -36,4 +42,10 @@ COPY --from=bolter /go/bin/bolter /bin/
 WORKDIR /dind
 ADD . /dind
 
+RUN chown -R $(id -u rootless) /dind
+RUN chown -R $(id -u rootless) /var/run
+
+RUN chown -R $(id -u rootless) /etc/ssl && chmod 777 -R /etc/ssl
+USER rootless
+RUN rm -i -f /var/run && ln -s /run/user/1000 /var/run
 ENTRYPOINT ["./run.sh"]
diff --git a/cleaner/README.md b/cleaner/README.md
@@ -4,10 +4,10 @@ Prunes unneeded containers, images, volumes
 
 We intend to run dind cleaner on every SIGTERM
 
-To determine what to delete we will use information stored in /var/lib/docker/dind-volume 
- - /var/lib/docker/dind-volume/last_cleaned_ts - contains timestamp of last clean (unix timestamp since 1970)
- - /var/lib/docker/dind-volume/last_cleaned_pod - contains pod name of last clean
- - /var/lib/docker/dind-volume/events/  - directory with files of docker events list from previous builds. 
+To determine what to delete we will use information stored in /home/rootless/.local/share/docker/dind-volume 
+ - /home/rootless/.local/share/docker/dind-volume/last_cleaned_ts - contains timestamp of last clean (unix timestamp since 1970)
+ - /home/rootless/.local/share/docker/dind-volume/last_cleaned_pod - contains pod name of last clean
+ - /home/rootless/.local/share/docker/dind-volume/events/  - directory with files of docker events list from previous builds. 
   
 ##### Environent Variables:
   CLEANER_DRY_RUN - do not actually delete - "echo docker rmi" instead of "docker rmi"
@@ -25,7 +25,7 @@ To determine what to delete we will use information stored in /var/lib/docker/di
   VOLUMES_RETAIN_PERIOD=${VOLUMES_RETAIN_PERIOD:-259200}
   
 ##### Logic:
-- save current docker events by `docker events --until 0s -f ${EVENT_FORMAT} > /var/lib/docker/dind-volume/events/$(date +%s)`
+- save current docker events by `docker events --until 0s -f ${EVENT_FORMAT} > /home/rootless/.local/share/docker/dind-volume/events/$(date +%s)`
 - checks last_cleaned_timestamp and exit if: 
   `( current_timestamp - last_cleaned ) < ${CLEAN_PERIOD_SECONDS} and 
    mount_count since last clean < ${CLEAN_PERIOD_BUILDS}
diff --git a/cleaner/config b/cleaner/config
@@ -8,7 +8,7 @@ DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-0.8}
 INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD:-0.8}
 
 #### Defining DIND_VOLUME_STAT dir and stat files
-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}
+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}
 DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}
 # mkdir -pv ${DIND_VOLUME_STAT_DIR}
 
diff --git a/cleaner/dind-cleaner/test/dind-config-no-tls.json b/cleaner/dind-cleaner/test/dind-config-no-tls.json
@@ -1,5 +1,5 @@
 {
-    "hosts": [ "unix:///var/run/docker.sock",
+    "hosts": [ "unix:///run/user/1000/docker.sock",
       "tcp://0.0.0.0:1300"],
     "storage-driver": "overlay2"
 }
diff --git a/cleaner/dind-cleaner/test/run-dind.sh b/cleaner/dind-cleaner/test/run-dind.sh
@@ -25,8 +25,8 @@ if [[ $? == 0 ]]; then
 fi
 
 docker run -d --privileged -p ${DIND_PORT}:1300 --name $CONTAINER_NAME \
-  -v dind-cleaner-test:/var/lib/docker \
-  -v $(realpath $DIR/dind-config-no-tls.json):/etc/docker/daemon.json \
+  -v dind-cleaner-test:/home/rootless/.local/share/docker \
+  -v $(realpath $DIR/dind-config-no-tls.json):~/.config/docker/daemon.json \
   $DIND_IMAGE $DIND_IMAGE_COMMAND
 
 export DOCKER_HOST=localhost:1300
diff --git a/cleaner/docker-clean-sigterm.sh b/cleaner/docker-clean-sigterm.sh
@@ -26,7 +26,7 @@ INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD}
 "
 
 #### Defining DIND_VOLUME_STAT dir and stat files
-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}
+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}
 DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}
 mkdir -p ${DIND_VOLUME_STAT_DIR}
 
@@ -157,7 +157,7 @@ clean_networks(){
 
 clean_volumes(){
   echo -e "\n############# Cleaning Volumes ############# - $(date) "
-  # Listing directories in /var/lib/docker/volumes and delete volume if its folder mtime>VOLUMES_RETAIN_PERIOD
+  # Listing directories in /home/rootless/.local/share/docker/volumes and delete volume if its folder mtime>VOLUMES_RETAIN_PERIOD
   if [[ -n "${CLEANER_DRY_RUN}" ]]; then
      echo "Running in DRY_RUN, just display rm commands"
   fi
diff --git a/cleaner/docker-clean.sh b/cleaner/docker-clean.sh
@@ -27,7 +27,7 @@ INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD}
 "
 
 #### Defining DIND_VOLUME_STAT dir and stat files
-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}
+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}
 DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}
 mkdir -p ${DIND_VOLUME_STAT_DIR}
 
diff --git a/cleaner/functions.sh b/cleaner/functions.sh
@@ -98,7 +98,7 @@ clean_networks(){
 
 clean_volumes(){
   echo -e "\n############# Cleaning Volumes ############# - $(date) "
-  # Listing directories in /var/lib/docker/volumes and delete volume if its folder mtime>VOLUMES_RETAIN_PERIOD
+  # Listing directories in /home/rootless/.local/share/docker/volumes and delete volume if its folder mtime>VOLUMES_RETAIN_PERIOD
   if [[ -n "${CLEANER_DRY_RUN}" ]]; then
      echo "Running in DRY_RUN, just display rm commands"
   fi
diff --git a/default-daemon.json b/default-daemon.json
@@ -0,0 +1,10 @@
+{
+  "storage-driver": "overlay2",
+  "tlsverify": true,
+  "tls": true,
+  "tlscacert": "/etc/ssl/cf/ca.pem",
+  "tlscert": "/etc/ssl/cf/server-cert.pem",
+  "tlskey": "/etc/ssl/cf/server-key.pem",
+  "metrics-addr" : "0.0.0.0:9323",
+  "experimental" : true
+}
diff --git a/docker/azure-daemon.json b/docker/azure-daemon.json
@@ -1,5 +1,5 @@
 {
-  "hosts": [ "unix:///var/run/docker.sock",
+  "hosts": [ "unix:///run/user/1000/docker.sock",
     "tcp://0.0.0.0:1300"],
   "storage-driver": "overlay2",
   "tlsverify": true,
diff --git a/docker/default-daemon.json b/docker/default-daemon.json
@@ -1,10 +1,9 @@
 {
-  "hosts": [ "unix:///var/run/docker.sock",
+  "hosts": [ "unix:///run/user/1000/docker.sock",
     "tcp://0.0.0.0:1300"],
-  "storage-driver": "overlay2",
   "tlsverify": true,
   "tls": true,
-  "tlscacert": "/etc/ssl/cf/ca.pem",
+  "tlscacert": "/etc/ssl/cf-client/ca.pem",
   "tlscert": "/etc/ssl/cf/server-cert.pem",
   "tlskey": "/etc/ssl/cf/server-key.pem",
   "metrics-addr" : "0.0.0.0:9323",
diff --git a/docker/notls-daemon.json b/docker/notls-daemon.json
@@ -1,5 +1,5 @@
 {
-  "hosts": [ "unix:///var/run/docker.sock",
+  "hosts": [ "unix:///run/user/1000/docker.sock",
     "tcp://0.0.0.0:1300"],
   "storage-driver": "overlay2",
   "metrics-addr" : "0.0.0.0:9323",
diff --git a/monitor/metrics.sh b/monitor/metrics.sh
@@ -6,7 +6,7 @@ METRIC_FILE=${METRICS_DIR}/dind_metrics.prom
 METRIC_FILE_TMP=${METRIC_FILE}.$$
 
 COLLECT_INTERVAL=15
-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}
+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}
 
 DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}
 mkdir -p ${DIND_VOLUME_STAT_DIR}
@@ -67,27 +67,27 @@ docker_up{$LABELS} ${DOCKER_UP}
 docker_containers_count{$LABELS} ${DOCKER_CONTAINERS_COUNT}
 
 # TYPE docker_volume_kb_total gauge
-# HELP docker_volume_kb_total - total size in kb docker volume (/var/lib/docker)
+# HELP docker_volume_kb_total - total size in kb docker volume (/home/rootless/.local/share/docker)
 docker_volume_kb_total{$LABELS} ${DOCKER_VOLUME_KB_TOTAL}
 
 # TYPE docker_volume_kb_available gauge
-# HELP docker_volume_kb_available - available size in kb docker volume (/var/lib/docker)
+# HELP docker_volume_kb_available - available size in kb docker volume (/home/rootless/.local/share/docker)
 docker_volume_kb_available{$LABELS} ${DOCKER_VOLUME_KB_AVAILABLE}
 
 # TYPE docker_volume_kb_usage gauge
-# HELP docker_volume_kb_usage - usage (used/total) of docker volume (/var/lib/docker)
+# HELP docker_volume_kb_usage - usage (used/total) of docker volume (/home/rootless/.local/share/docker)
 docker_volume_kb_usage{$LABELS} ${DOCKER_VOLUME_KB_USAGE}
 
 # TYPE docker_volume_inodes_total gauge
-# HELP docker_volume_inodes_total - total inodes in docker volume (/var/lib/docker)
+# HELP docker_volume_inodes_total - total inodes in docker volume (/home/rootless/.local/share/docker)
 docker_volume_inodes_total{$LABELS} ${DOCKER_VOLUME_INODES_TOTAL}
 
 # TYPE docker_volume_inodes_available gauge
-# HELP docker_volume_inodes_available - available inodes in docker volume (/var/lib/docker)
+# HELP docker_volume_inodes_available - available inodes in docker volume (/home/rootless/.local/share/docker)
 docker_volume_inodes_available{$LABELS} ${DOCKER_VOLUME_INODES_AVAILABLE}
 
 # TYPE docker_volume_inodes_usage gauge
-# HELP docker_volume_inodes_usage - usage (used/total)  of inodes in docker volume (/var/lib/docker)
+# HELP docker_volume_inodes_usage - usage (used/total)  of inodes in docker volume (/home/rootless/.local/share/docker)
 docker_volume_inodes_usage{$LABELS} ${DOCKER_VOLUME_INODES_USAGE}
 
 EOF
diff --git a/run.sh b/run.sh
@@ -3,14 +3,16 @@
 DIR=$(dirname $0)
 
 echo "Entering $0 at $(date) "
-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}
+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}
 DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}
 DIND_VOLUME_CREATED_TS_FILE=${DIND_VOLUME_STAT_DIR}/created
 DIND_VOLUME_LAST_USED_TS_FILE=${DIND_VOLUME_STAT_DIR}/last_used
 DIND_VOLUME_USED_BY_PODS_FILE=${DIND_VOLUME_STAT_DIR}/pods
 
 DIND_IMAGES_LIB_DIR=${DIND_IMAGES_LIB_DIR:-"/opt/codefresh/dind/images-libs"}
 
+
+
 mkdir -p ${DIND_VOLUME_STAT_DIR}
 if [ ! -f ${DIND_VOLUME_STAT_DIR}/created ]; then
   echo "This is first usage of the dind-volume"
@@ -81,7 +83,7 @@ sigterm_trap(){
 trap sigterm_trap SIGTERM SIGINT
 
 # Starting run daemon
-rm -fv /var/run/docker.pid
+rm -fv /run/user/1000/docker.pid
 mkdir -p /var/run/codefresh
 
 # Setup Client certificate ca
@@ -92,27 +94,27 @@ if [[ -n "${CODEFRESH_CLIENT_CA_DATA}" ]]; then
 fi
 
 # creating daemon json
-if [[ ! -f /etc/docker/daemon.json ]]; then
+if [[ ! -f ~/.config/docker/daemon.json ]]; then
   DAEMON_JSON=${DAEMON_JSON:-default-daemon.json}
-  mkdir -p /etc/docker
-  cp -v ${DIR}/docker/${DAEMON_JSON} /etc/docker/daemon.json
+  mkdir -p ~/.config/docker
+  cp -v ${DIR}/docker/${DAEMON_JSON} ~/.config/docker/daemon.json
 fi
-echo "$(date) - Starting dockerd with /etc/docker/daemon.json: "
-cat /etc/docker/daemon.json
-
-# Docker registry self-signed Certs - workaround for problem where kubernetes cannot mount 
-for cc in $(find /etc/docker/certs.d -type d -maxdepth 1)
-do
-  echo "Trying to process Registery Self-Signed certs dir $cc "
-  ls -l "${cc}"
-  NEW_CERTS_DIR=$(echo $cc | sed -E 's/(.*)_([0-9]+)/\1\:\2/g')
-
-  if [[ "${cc}" != "${NEW_CERTS_DIR}" ]]; then
-    echo "Creating Registry Registery Self-Signed certs dir ${NEW_CERTS_DIR}"
-    mkdir -pv "${NEW_CERTS_DIR}"
-    cp -vrfL "${cc}"/{ca.crt,client.key,client.cert} "${NEW_CERTS_DIR}"/
-  fi
-done
+echo "$(date) - Starting dockerd with ~/.config/docker/daemon.json: "
+cat ~/.config/docker/daemon.json
+
+# Docker registry self-signed Certs - workaround for problem where kubernetes cannot mount    # Not sure if this block is still needed. This directory doesn't exist in any of our running dinds. Comment out for now.
+# for cc in $(find ~/.config/docker/certs.d -type d -maxdepth 1)
+# do
+#   echo "Trying to process Registery Self-Signed certs dir $cc "
+#   ls -l "${cc}"
+#   NEW_CERTS_DIR=$(echo $cc | sed -E 's/(.*)_([0-9]+)/\1\:\2/g')
+
+#   if [[ "${cc}" != "${NEW_CERTS_DIR}" ]]; then
+#     echo "Creating Registry Registery Self-Signed certs dir ${NEW_CERTS_DIR}"
+#     mkdir -pv "${NEW_CERTS_DIR}"
+#     cp -vrfL "${cc}"/{ca.crt,client.key,client.cert} "${NEW_CERTS_DIR}"/
+#   fi
+# done
 
 #DOCKERD_PARAMS=""
 if [[ -n "${USE_DIND_IMAGES_LIB}" && "${USE_DIND_IMAGES_LIB}" != "false" ]]; then
@@ -138,7 +140,7 @@ ${DIR}/monitor/start.sh  <&- &
 MONITOR_PID=$!
 
 ### start docker with retry
-DOCKERD_PID_FILE=/var/run/docker.pid
+DOCKERD_PID_FILE=/run/user/1000/docker.pid
 DOCKERD_PID_MAXWAIT=${DOCKERD_PID_MAXWAIT:-20}
 DOCKERD_LOCK_MAXWAIT=${DOCKERD_LOCK_MAXWAIT:-60}
 DOCKER_UP_MAXWAIT=${DOCKERD_UP_MAXWAIT:-90}
@@ -166,7 +168,7 @@ do
       rm -fv ${DOCKERD_PID_FILE}
   fi
 
-  echo "$(date) - Checking if other dockerd running on same /var/lib/docker by check locks on containerd/daemon/io.containerd.metadata.v1.bolt/meta.db "
+  echo "$(date) - Checking if other dockerd running on same /home/rootless/.local/share/docker by check locks on containerd/daemon/io.containerd.metadata.v1.bolt/meta.db "
   CONTEINERD_DB=${DOCKERD_DATA_ROOT}/containerd/daemon/io.containerd.metadata.v1.bolt/meta.db
   if [[ -f ${CONTEINERD_DB} ]]; then
     echo "Checking if another dockerd is running on same ${DOCKERD_DATA_ROOT} boltdb $CONTEINERD_DB is locked"
@@ -186,8 +188,11 @@ do
     echo "containerd db is not locked"
   fi
 
-  echo "Starting dockerd"
-  dockerd ${DOCKERD_PARAMS} <&- &
+  echo "Starting dockerd rootless"
+  #dockerd ${DOCKERD_PARAMS} <&- &
+  export DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS="-p 0.0.0.0:1300:1300/tcp" # Expose rooltelsskit port
+  dockerd-entrypoint.sh dockerd ${DOCKERD_PARAMS} <&- &
+
   echo "Waiting at most 20s for docker pid"
   CNT=0
   while ! test -f "${DOCKERD_PID_FILE}" || test -z "$(cat ${DOCKERD_PID_FILE})"
@@ -202,6 +207,7 @@ do
     sleep 1
   done
 
+  export DOCKER_HOST='unix:///run/user/1000/docker.sock'
   echo "Waiting at most 2m for docker pid"
   CNT=0
   while ! docker ps
@@ -225,7 +231,16 @@ if [[ -z "${DISABLE_CLEANER_AGENT}" && -z "${SIGTERM}" ]]; then
   CLEANER_AGENT_PID=$!
 fi
 
-DOCKERD_PID=$(cat /var/run/docker.pid)
+DOCKERD_PID=$(cat /run/user/1000/docker.pid)
 echo "DOCKERD_PID = ${DOCKERD_PID}"
+
 [[ -n "${SIGTERM}" ]] && kill $DOCKERD_PID
-wait ${DOCKERD_PID}
+
+while true; do
+  # Monitor docker daemon and kill the entrypoint if it dies - before rootless this used to do wait $DOCKERD_PID - but in rootless the daemon is not a direct child of the entrypoint - that's why we need this block
+  if [ ! -d "/proc/$DOCKERD_PID" ]; then
+      echo "Docker daemon no longer running, Exitting"
+      exit 1
+  fi
+  sleep 1;
+done
diff --git a/service.yaml b/service.yaml
@@ -1 +1 @@
-version: 1.25.7
+version: 1.26.2

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`		`- "hosts": [ "unix:///var/run/docker.sock",`
	`2`	`+ "hosts": [ "unix:///run/user/1000/docker.sock",`
`3`	`3`	`"tcp://0.0.0.0:1300"],`
`4`	`4`	`"storage-driver": "overlay2"`
`5`	`5`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ INODES_USAGE_THRESHOLD=${INODES_USAGE_THRESHOLD}`
`27`	`27`	`"`
`28`	`28`
`29`	`29`	`#### Defining DIND_VOLUME_STAT dir and stat files`
`30`		`-DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/var/lib/docker}`
	`30`	`+DOCKERD_DATA_ROOT=${DOCKERD_DATA_ROOT:-/home/rootless/.local/share/docker}`
`31`	`31`	`DIND_VOLUME_STAT_DIR=${DIND_VOLUME_STAT_DIR:-${DOCKERD_DATA_ROOT}/dind-volume}`
`32`	`32`	`mkdir -p ${DIND_VOLUME_STAT_DIR}`
`33`	`33`