Refactor kci-deploy for improved reliability, structure, and full Docker-in-Docker deployment

localinstall/.gitignore

@@ -0,0 +1,7 @@
+kernelci
+config/out/*
+.sudo_as_admin_successful
+.cache
+.local
+.docker
+.bash_history

localinstall/Containerfile

@@ -0,0 +1,48 @@
+FROM kernelci/kernelci:latest
+
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+USER root
+
+# Install dependencies for Docker installation
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    sudo \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release
+
+# Add Docker's official GPG key
+RUN mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg
+
+# Set up Docker repository (assuming Debian-based image)
+RUN echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+    $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+# Install Docker Engine, CLI, and Compose plugin
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    docker-ce \
+    docker-ce-cli \
+    containerd.io \
+    docker-compose-plugin \
+    expect
+
+# Make sure a user exists with the same USER_ID/GROUP_ID as the host user
+# to allow access to the host docker socket
+RUN groupadd -g ${GROUP_ID} kernelci || true && \
+    useradd -u ${USER_ID} -g ${GROUP_ID} -m -s /bin/bash kernelci || true
+
+# Add the user to the sudoers
+RUN usermod -aG sudo kernelci && \
+    echo "kernelci ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+USER kernelci
+WORKDIR /home/kernelci
+
+ENTRYPOINT ["/bin/bash", "./scripts/run.sh"]

localinstall/README.md

@@ -1,16 +1,24 @@
-# kci-easy
-
+# kci-deploy
 Get your own KernelCI instance up and running in no time.
 
-## Getting started
-
-### Prerequisites
+## Prerequisites
+- Docker
 
-- git
-- Docker (with `compose` plugin, set up for a regular user)
-- Python environment with [KernelCI core dependencies](https://github.com/kernelci/kernelci-core/blob/main/requirements.txt) installed
-- expect
+## Configure
+Configure and setup credentials in config files located in `config` folder.
 
-### Running
+## Run
+You can deploy your KernelCI deployment by simply executing:
+```bash
+./kci-deploy.sh deploy
+```
 
-Change `ADMIN_PASSWORD` in the `main.cfg`, then run shell scripts from the root directory in their order.
+You can stop your local deployment by executing:
+```bash
+./kci-deploy.sh stop
+```
+and
+```bash
+./kci-deploy.sh start
+```
+to start it again.

localinstall/kci-deploy.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+
+IMAGE_NAME="local/kernelci-deployer:latest"
+BUILD_IMAGE=false
+ACTION=""
+CONTAINER_ARGS=()
+
+function print_help() {
+  echo "Usage: $0 [--build] (deploy|start|stop) [args...]"
+  echo
+  echo "Options:"
+  echo "  --build     Force rebuild of the Deployer image (optional)"
+  echo "  deploy      Configure and start the kernelci deployment"
+  echo "  start       Start the already configured kernelci deployment (default if no action specified)"
+  echo "  stop        Stop the kernelci deployment"
+  echo "  -h, --help  Show this help message"
+  echo
+  echo "Arguments after 'deploy', 'start' or 'stop' are passed to the container entrypoint"
+  exit 0
+}
+
+# Parse args
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --build)
+      BUILD_IMAGE=true
+      shift
+      ;;
+    deploy|start|stop)
+      if [[ -n "$ACTION" ]]; then
+        echo "Error: Cannot use more than one command among 'deploy', 'start' or 'stop'"
+        exit 1
+      fi
+      ACTION=$1
+      shift
+      CONTAINER_ARGS=("$@")
+      break
+      ;;
+    -h|--help)
+      print_help
+      ;;
+    *)
+      echo "Unknown option: $1"
+      print_help
+      ;;
+  esac
+done
+
+# Default
+if [[ -z "$ACTION" ]]; then
+  ACTION="start"
+fi
+
+USER_ID=$(id -u)
+GROUP_ID=$(id -g)
+
+if [[ "$BUILD_IMAGE" = true || -z $(docker images -q "$IMAGE_NAME") ]]; then
+  echo "Building $IMAGE_NAME"
+  docker build \
+    --build-arg USER_ID=$USER_ID \
+    --build-arg GROUP_ID=$GROUP_ID \
+    -f Containerfile \
+    -t "$IMAGE_NAME" \
+    .
+fi
+
+echo "Running $IMAGE_NAME with action '$ACTION' and args: ${CONTAINER_ARGS[*]}"
+docker run --rm \
+  --name kernelci-deployer \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v "$(pwd)":"$(pwd)" \
+  --workdir "$(pwd)" \
+  --group-add "$(stat -c '%g' /var/run/docker.sock)" \
+  --network host \
+  "$IMAGE_NAME" \
+  "$ACTION" "${CONTAINER_ARGS[@]}"

localinstall/1-rebuild_all.sh → localinstall/scripts/1-rebuild_all.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
-. ./main.cfg
+
+. ./config/main.cfg
+
+set -e
 
 # i am groot?
 if [ $(id -u) -ne 0 ]; then
@@ -8,13 +11,6 @@ else
     SUDO=
 fi
 
-function failonerror {
-    if [ $? -ne 0 ]; then
-        echo "Failed"
-        exit 1
-    fi
-}
-
 # if directry kernelci doesn't exist, then we dont have repos cloned
 if [ ! -d kernelci ]; then
     echo Create kernelci directory, clone repos and checkout branches
@@ -86,32 +82,14 @@ core_url=$(git remote get-url origin)
 build_args="--build-arg pipeline_rev=$pipeline_rev --build-arg core_rev=$core_rev --build-arg api_rev=$api_rev --build-arg pipeline_url=$pipeline_url --build-arg core_url=$core_url --build-arg api_url=$api_url"
 px_arg='--prefix=local/staging-'
 args="build --verbose $px_arg $build_args"
-echo Build docker images: kernelci args=$args
-./kci docker $args kernelci 
-failonerror
-echo Build docker images: k8s+kernelci
-./kci docker $args k8s kernelci
-failonerror
 echo Build docker images: api
-./kci docker $args kernelci api --version="$api_rev"
-failonerror
+./kci docker $args kernelci api
 echo Build docker images: pipeline
-./kci docker $args kernelci pipeline --version="$pipeline_rev"
-failonerror
-echo Tag docker image of api to latest
-docker tag local/staging-kernelci:api-$api_rev local/staging-kernelci:api
-failonerror
-echo Tag docker image of pipeline to latest
-docker tag local/staging-kernelci:pipeline-$pipeline_rev local/staging-kernelci:pipeline
-failonerror
+./kci docker $args kernelci pipeline
 echo Build docker images: clang-17+kselftest+kernelci for x86
 ./kci docker $args clang-17 kselftest kernelci --arch x86
-failonerror
 echo Build docker images: gcc-12+kselftest+kernelci for x86
 ./kci docker $args gcc-12 kselftest kernelci --arch x86
-failonerror
 echo Build docker images: gcc-12+kselftest+kernelci for arm64
 ./kci docker $args gcc-12 kselftest kernelci --arch arm64
-failonerror
-
 

localinstall/scripts/2-prepare_api.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. ./config/main.cfg
+
+set -e
+
+# i am groot?
+if [ $(id -u) -ne 0 ]; then
+    SUDO=sudo
+else
+    SUDO=
+fi
+
+cp config/.env-api kernelci/kernelci-api/.env
+cp config/api-configs.yaml kernelci/kernelci-core/config/core/
+cp config/kernelci-cli.toml kernelci/kernelci-core/kernelci.toml
+
+sed -i "s/#SECRET_KEY=/SECRET_KEY=${API_SECRET_KEY}/" kernelci/kernelci-api/.env
+
+cd kernelci/kernelci-api
+mkdir -p docker/redis/data
+${SUDO} chmod -R 0777 docker/storage/data
+${SUDO} chmod -R 0777 docker/redis/data
+# enable ssh and storage nginx
+mkdir -p ../../config/out
+sed -i 's/^#  /  /' docker-compose.yaml
+if [ -f ../../config/out/ssh.key ]; then
+    echo "ssh.key already exists"
+else
+    # generate non-interactively ssh key to ssh.key
+    ssh-keygen -t rsa -b 4096 -N "" -f ../../config/out/ssh.key
+fi
+# get public key and add to docker/ssh/user-data/authorized_keys
+cat ../../config/out/ssh.key.pub > docker/ssh/user-data/authorized_keys

localinstall/scripts/3-start_api.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. ./config/main.cfg
+
+set -e
+
+# i am groot?
+if [ $(id -u) -ne 0 ]; then
+    SUDO=sudo
+else
+    SUDO=
+fi
+
+cd kernelci/kernelci-api
+
+# down, just in case old containers are running
+docker compose down
+docker compose up -d
+echo "Waiting for API to be up"
+sleep 5
+# loop until the API is up, try 5 times
+i=0
+while [ $i -lt 5 ]; do
+    ANSWER=$(curl http://localhost:8001/latest/)
+    # must be {"message":"KernelCI API"}
+    if [ "$ANSWER" != "{\"message\":\"KernelCI API\"}" ]; then
+        echo "API is not up"
+        i=$((i+1))
+        sleep 5
+    else
+        echo "API is up"
+        break
+    fi
+done
+