containers · cgwalters · Aug 5, 2025 · Jul 16, 2025 · Jul 15, 2025 · Jul 16, 2025
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -41,10 +41,12 @@ jobs:
 
     - name: Setup /dev/kvm
       run: |
+        set -eux
         echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules
+        echo 'KERNEL=="vhost-vsock", GROUP="kvm", MODE="0666", OPTIONS+="static_node=vhost-vsock"' | sudo tee /etc/udev/rules.d/99-vhost-vsock.rules
         sudo udevadm control --reload-rules
-        sudo udevadm trigger --name-match=kvm --settle
-        ls -l /dev/kvm
+        sudo udevadm trigger --settle
+        ls -l /dev/kvm /dev/vhost-vsock
 
     - name: Install dependencies
       run: |
@@ -55,12 +57,9 @@ jobs:
           erofs-utils \
           fsverity \
           mtools \
-          libvirt-daemon \
-          libvirt-daemon-driver-qemu \
-          python3-libvirt \
-          qemu-system \
+          python3-pytest-asyncio \
+          qemu-kvm \
           systemd-boot-efi
-        sudo apt-get build-dep systemd e2fsprogs
 
     - name: Get a newer podman for heredoc support (from plucky)
       run: |
@@ -70,10 +69,23 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - name: Install patched tools
+    - name: Check cache for patched tools
+      uses: actions/cache@v3
+      with:
+        path: ~/bin
+        key: patched-tools-bin-${{ hashFiles('examples/common/install-patched-tools') }}
+
+    - name: Ensure patched tools are installed
       run: |
-        mkdir ~/bin
-        examples/common/install-patched-tools ~/bin
+        set -eux
+        if [ ! -x "$HOME/bin/mkfs.ext4" ]; then
+          sudo apt-get build-dep systemd e2fsprogs
+          mkdir ~/bin
+          examples/common/install-patched-tools ~/bin
+        fi
+
+    - name: Install systemd-ssh-proxy polyfill
+      run: sudo cp examples/bls/test-thing.workarounds/systemd-ssh-proxy /usr/lib/systemd
 
     - name: Run example tests
       run: |

diff --git a/examples/.gitignore b/examples/.gitignore
@@ -1,8 +1,9 @@
+/*/*.qcow2
 /*/VARS_CUSTOM.secboot.fd*
 /*/cfsctl
 /*/extra/usr/lib/dracut/modules.d/37composefs/composefs-setup-root
-/*/*.qcow2
 /*/secureboot
 /*/tmp/
 /common/fix-verity/fix-verity.efi
 /test/bots
+__pycache__/
diff --git a/examples/README.md b/examples/README.md
@@ -12,3 +12,45 @@ a verified operating system image.
  - `unified`: similar to the `uki` example, but avoiding the intermediate `cfsctl` step by running `cfsctl` inside a build stage from the `Containerfile` itself.
    This involves bind-mounting the earlier build stage of the base image so that we can measure it from inside the stage that builds the UKI.
  - `unified-secureboot`: based on the `unified` example, adding signing for Secure Boot.
+
+## Using the examples
+
+The main use of the examples is to act as a scratch space for feature
+development (like initramfs integration) and to show how you can build a system
+image in various configurations using composefs.  They are also run from CI and
+are very useful for local testing, however.
+
+You can build the various images using the `build` script found in each
+subdirectory.  It takes a single argument: the OS to build the image from
+(`fedora`, `rawhide`, `arch`, `ubuntu`, `rhel9`, etc.).  You should not build
+multiple images in parallel due to conflicting feature flags and shared use of
+the tmp/ directory.  After the image is built, you can run tests against it by
+saying something like:
+
+```
+TEST_IMAGE=examples/bls/arch-bls-efi.qcow2 pytest examples/test
+```
+
+Building and running tests on a particular image is supported via the
+`examples/test/run` script, which you can use like:
+
+```
+examples/test/run bls rhel9
+```
+
+The tests are run using [`test.thing`](https://codeberg.org/lis/test.thing). We
+keep a copy of it in-tree.  You you can also use it to run the VM images for
+manual inspection:
+
+```
+examples/testthing.py examples/bls/fedora-bls-efi.qcow2
+```
+
+In that case, you should add this fragment to your ssh configuration:
+
+```
+Host tt.*
+        ControlPath ${XDG_RUNTIME_DIR}/test.thing/%h/ssh
+```
+
+So you can access the test machine via `ssh tt.0` and so on.
diff --git a/examples/bls/Containerfile b/examples/bls/Containerfile
@@ -1,5 +1,4 @@
 FROM fedora:42
-COPY extra /
 RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
     set -eux
 
@@ -12,9 +11,16 @@ RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
         strace \
         util-linux \
         systemd
+EOF
 
+# --- Everything above this line should hopefully stay cached ---
+
+COPY cfsctl /usr/bin
+COPY extra /
+COPY test-thing.workarounds/fedora-42 /
+RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
+    kernel-install add-all
     systemctl enable systemd-networkd
     passwd -d root
     mkdir /sysroot
 EOF
-COPY cfsctl /usr/bin
diff --git a/examples/bls/Containerfile.arch b/examples/bls/Containerfile.arch
@@ -1,11 +1,8 @@
 FROM archlinux AS base
-COPY extra /
-RUN <<EOF
+RUN --mount=type=cache,target=/var/cache/pacman/pkg \
+    --mount=type=cache,target=/var/lib/pacman/sync <<EOF
     set -eux
 
-    touch /etc/machine-id
-    echo 'root=/dev/vda2' > /etc/kernel/cmdline
-
     pacman -Syu --noconfirm
     pacman -Sy --noconfirm \
         btrfs-progs \
@@ -16,11 +13,19 @@ RUN <<EOF
         openssh \
         skopeo \
         strace
+EOF
+
+# --- Everything above this line should hopefully stay cached ---
 
-    kernel-install add "$(ls /usr/lib/modules)" /usr/lib/modules/"$(ls /usr/lib/modules)"/vmlinuz
+COPY cfsctl /usr/bin
+COPY extra /
+RUN <<EOF
+    touch /etc/machine-id
+    echo 'root=/dev/vda2' > /etc/kernel/cmdline
+
+    kernel-install add-all
 
     systemctl enable systemd-networkd systemd-resolved sshd
     passwd -d root
     mkdir /sysroot
 EOF
-COPY cfsctl /usr/bin
diff --git a/examples/bls/Containerfile.rawhide b/examples/bls/Containerfile.rawhide
@@ -1,5 +1,4 @@
 FROM fedora:rawhide
-COPY extra /
 RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
     set -eux
 
@@ -12,9 +11,15 @@ RUN --mount=type=cache,target=/var/cache/libdnf5 <<EOF
         strace \
         systemd \
         util-linux
+EOF
+
+# --- Everything above this line should hopefully stay cached ---
 
+COPY cfsctl /usr/bin
+COPY extra /
+RUN <<EOF
     systemctl enable systemd-networkd
     passwd -d root
+    kernel-install add-all
     mkdir /sysroot
 EOF
-COPY cfsctl /usr/bin
diff --git a/examples/bls/Containerfile.rhel9 b/examples/bls/Containerfile.rhel9
@@ -1,13 +1,7 @@
 # FROM docker.io/redhat/ubi9  missing: dosfstools, kernel
 FROM quay.io/centos/centos:9
-COPY extra /
 RUN --mount=type=cache,target=/var/cache/dnf <<EOF
     set -eux
-    mkdir -p /etc/kernel
-    touch /etc/kernel/cmdline
-
-    echo layout=bls | tee /etc/kernel/install.conf
-
     dnf --setopt keepcache=1 install --allowerasing -y \
         NetworkManager \
         composefs \
@@ -19,9 +13,22 @@ RUN --mount=type=cache,target=/var/cache/dnf <<EOF
         strace \
         systemd \
         util-linux
+EOF
+
+# --- Everything above this line should hopefully stay cached ---
+
+COPY cfsctl /usr/bin
+COPY extra /
+COPY test-thing.workarounds/rhel9 /
+RUN <<EOF
+    set -eux
+    mkdir -p /etc/kernel
+    touch /etc/kernel/cmdline
+
+    echo layout=bls | tee /etc/kernel/install.conf
+    kernel-install add $(ls /usr/lib/modules) /usr/lib/modules/*/vmlinuz
 
     systemctl enable tmp.mount
     passwd -d root
     mkdir /sysroot
 EOF
-COPY cfsctl /usr/bin
diff --git a/examples/bls/Containerfile.ubuntu b/examples/bls/Containerfile.ubuntu
@@ -35,4 +35,5 @@ RUN <<EOF
     passwd -d root
     mkdir /sysroot
 EOF
+COPY test-thing.workarounds/debian /
 COPY cfsctl /usr/bin
diff --git a/examples/bls/build b/examples/bls/build
@@ -44,6 +44,7 @@ cp ../../target/release/composefs-setup-root extra/usr/lib/dracut/modules.d/37co
 CFSCTL='./cfsctl --repo tmp/sysroot/composefs'
 
 rm -rf tmp
+rm -rf tmp/efi tmp/sysroot/composefs/images
 mkdir -p tmp/sysroot/composefs
 
 podman build \
@@ -59,7 +60,7 @@ if [ "${FS_VERITY_MODE:-repart}" = "none" ]; then
     CFSCTL="$CFSCTL --insecure"
 fi
 
-${CFSCTL} oci prepare-boot "${BASE_ID}" --bootdir tmp/efi --cmdline console=ttyS0,115200 --entry-id=example --cmdline rw
+${CFSCTL} oci prepare-boot "${BASE_ID}" --bootdir tmp/efi --entry-id=example --cmdline rw
 
 ../common/install-systemd-boot
 ../common/make-image "${os}-bls-efi.qcow2"
diff --git a/examples/bls/extra/root/.ssh/authorized_keys b/examples/bls/extra/root/.ssh/authorized_keys
diff --git a/examples/bls/test-thing.workarounds/README.md b/examples/bls/test-thing.workarounds/README.md
@@ -0,0 +1,18 @@
+# Guest support workarounds
+
+These are extra files that you can add to virtual machine guests to enable
+support for missing features required by `test.thing`:
+
+ - [debian/](debian/): enables ephemeral ssh key support
+ - [fedora-42/](fedora-42/): enables ephemeral ssh key support ([this is
+   supported without a workaround in Fedora 43 and
+   later](https://src.fedoraproject.org/rpms/openssh/pull-request/101))
+ - [rhel9/](rhel9/): enables sshd vsock listener (with ephemeral ssh key
+   support) and sends the expected `sd_notify` message when the guest reaches
+   `multi-user.target`
+
+`test.thing` can also work with guests lacking support for ephemeral ssh keys
+by including a fixed ssh key in the image, or by using the
+`ssh.authorized_keys.root` credential (since systemd 252), but this requires
+modifying root's home directory at runtime and only works if another ssh key
+isn't already present, so it isn't enabled by default.
diff --git a/examples/bls/test-thing.workarounds/debian/etc/default/ssh b/examples/bls/test-thing.workarounds/debian/etc/default/ssh
@@ -0,0 +1,2 @@
+# Debian
+SSHD_OPTS=-o "AuthorizedKeysFile /run/credentials/@system/ssh.ephemeral-authorized_keys-all .ssh/authorized_keys"
diff --git a/examples/bls/test-thing.workarounds/fedora-42/etc/sysconfig/sshd b/examples/bls/test-thing.workarounds/fedora-42/etc/sysconfig/sshd
@@ -0,0 +1,3 @@
+# Needed for Fedora 42
+# https://src.fedoraproject.org/rpms/openssh/pull-request/101
+OPTIONS=-o "AuthorizedKeysFile /run/credentials/@system/ssh.ephemeral-authorized_keys-all .ssh/authorized_keys"
diff --git a/examples/bls/test-thing.workarounds/rhel9/etc/notify-multiuser.py b/examples/bls/test-thing.workarounds/rhel9/etc/notify-multiuser.py
@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+
+"""Notify that the system has reached multi-user.target."""
+
+import os
+import socket
+from pathlib import Path
+
+credentials = Path(os.environ["CREDENTIALS_DIRECTORY"])
+notify_socket = (credentials / "vmm.notify_socket").read_text()
+af, cid, port = notify_socket.split(":")
+assert af == "vsock"
+sock = socket.socket(socket.AF_VSOCK, socket.SOCK_SEQPACKET)
+sock.connect((int(cid), int(port)))
+sock.sendmsg([b"X_SYSTEMD_UNIT_ACTIVE=multi-user.target\n"])
diff --git a/...ing.workarounds/rhel9/etc/systemd/system/multi-user.target.wants/notify-multiuser.service b/...ing.workarounds/rhel9/etc/systemd/system/multi-user.target.wants/notify-multiuser.service
@@ -0,0 +1 @@
+../notify-multiuser.service
diff --git a/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/notify-multiuser.service b/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/notify-multiuser.service
@@ -0,0 +1,9 @@
+[Unit]
+After=multi-user.target
+Wants=multi-user.target
+
+[Service]
+LoadCredential=vmm.notify_socket
+ExecStart=/etc/notify-multiuser.py
+Type=exec
+RemainAfterExit=yes
diff --git a/...ls/test-thing.workarounds/rhel9/etc/systemd/system/sockets.target.wants/sshd-vsock.socket b/...ls/test-thing.workarounds/rhel9/etc/systemd/system/sockets.target.wants/sshd-vsock.socket
@@ -0,0 +1 @@
+../sshd-vsock.socket
diff --git a/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/sshd-vsock.socket b/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/sshd-vsock.socket
@@ -0,0 +1,7 @@
+[Unit]
+Wants=ssh-access.target
+Before=ssh-access.target
+
+[Socket]
+ListenStream=vsock::22
+Accept=yes
diff --git a/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/[email protected] b/examples/bls/test-thing.workarounds/rhel9/etc/systemd/system/[email protected]
@@ -0,0 +1,11 @@
+[Unit]
+Description=OpenSSH per-connection server daemon
+Documentation=man:sshd(8) man:sshd_config(5)
+Wants=sshd-keygen.target
+After=sshd-keygen.target
+
+[Service]
+EnvironmentFile=-/etc/sysconfig/sshd
+ExecStart=-/usr/sbin/sshd -i $OPTIONS -o "AuthorizedKeysFile ${CREDENTIALS_DIRECTORY}/ssh.ephemeral-authorized_keys-all .ssh/authorized_keys"
+StandardInput=socket
+LoadCredential=ssh.ephemeral-authorized_keys-all
diff --git a/examples/bls/test-thing.workarounds/systemd-ssh-proxy b/examples/bls/test-thing.workarounds/systemd-ssh-proxy
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+"""Polyfill for systemd-ssh-proxy on systems that don't have it."""
+
+import argparse
+import socket
+import sys
+
+parser = argparse.ArgumentParser(description="systemd-ssh-proxy polyfill")
+parser.add_argument("addr", type=lambda value: int(value.removeprefix("vsock/")))
+parser.add_argument("port", type=int)
+args = parser.parse_args()
+
+stdout = socket.socket(fileno=sys.stdout.fileno())
+
+vsock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+vsock.connect((args.addr, args.port))
+socket.send_fds(stdout, [b"\0"], [vsock.fileno()])
+vsock.close()
diff --git a/examples/common/install-systemd-boot b/examples/common/install-systemd-boot
@@ -3,7 +3,7 @@
 set -eux
 
 mkdir -p tmp/efi/loader
-echo 'timeout 3' > tmp/efi/loader/loader.conf
+echo 'timeout 1' > tmp/efi/loader/loader.conf
 mkdir -p tmp/efi/EFI/BOOT tmp/efi/EFI/systemd
 cp /usr/lib/systemd/boot/efi/systemd-bootx64.efi tmp/efi/EFI/systemd
 cp /usr/lib/systemd/boot/efi/systemd-bootx64.efi tmp/efi/EFI/BOOT/BOOTX64.EFI
diff --git a/examples/pyproject.toml b/examples/pyproject.toml
@@ -0,0 +1,5 @@
+[tool.pytest.ini_options]
+addopts = "--strict-markers -v"
+asyncio_mode = "auto"
+pythonpath = "."
+testpaths = ["test"]
diff --git a/examples/test/run b/examples/test/run
@@ -4,16 +4,8 @@ set -eux
 
 cd "${0%/*}/.."
 
-if [ ! -e test/bots ]; then
-    if [ -h ~/.config/cockpit-dev/bots ]; then
-        ln -sfT "$(realpath --relative-to=test ~/.config/cockpit-dev)/bots" test/bots
-    else
-        git clone https://github.com/cockpit-project/bots test/bots
-    fi
-fi
-
 EXAMPLE="$1"
 OS="$2"
 
 "${EXAMPLE}/build" "${OS}"
-test/run-tests "${EXAMPLE}/${OS}-${EXAMPLE}-efi.qcow2"
+TEST_IMAGE="${EXAMPLE}/${OS}-${EXAMPLE}-efi.qcow2" pytest test
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Debian
		SSHD_OPTS=-o "AuthorizedKeysFile /run/credentials/@system/ssh.ephemeral-authorized_keys-all .ssh/authorized_keys"