Skip to content

Commit 51256ab

Browse files
mahyarsamanimahyarsamani
committed
resources: Scripts to build disk with gem5 driver
This change adds scripts to build disk-images that include an installation of gem5-bridge kernel object so that m5 memory can be mapped without the need for sudo access. This is necessary when benchmarks should not be run with sudo permission, e.g. when using MPI runtimes. Additionally, it further breaks down the installation steps to more granular steps that are easier to understand and potentially debug.
1 parent 6f8d494 commit 51256ab

33 files changed

+468
-256
lines changed

‎src/ubuntu-generic-diskimages/BUILDING.md renamed to ‎src/ubuntu-generic-disk-images/BUILDING.md

File renamed without changes.

‎src/ubuntu-generic-diskimages/README.md renamed to ‎src/ubuntu-generic-disk-images/README.md

File renamed without changes.

‎src/ubuntu-generic-diskimages/build-arm.sh renamed to ‎src/ubuntu-generic-disk-images/build-arm.sh

+16-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,21 @@ if [[ "$ubuntu_version" != "22.04" && "$ubuntu_version" != "24.04" ]]; then
2929
exit 1
3030
fi
3131

32-
# make the flash0.sh file
32+
# Create a dictionary to store the path to the kernel modules for each Ubuntu version
33+
declare -A kernel_modules_paths
34+
kernel_modules_paths["22.04"]="modules/u2204"
35+
kernel_modules_paths["24.04"]="modules/u2404/"
36+
37+
# Get the path to the kernel modules for the specified Ubuntu version
38+
kernel_modules_path=${kernel_modules_paths[$ubuntu_version]}
39+
pushd $kernel_modules_path
40+
./copy_modules.sh
41+
popd
42+
43+
# Store the image name from the second command line argument or default to "arm-ubuntu"
44+
image_name="${2:-arm-ubuntu}"
45+
46+
# make the flash0.img file
3347
cd ./files
3448
dd if=/dev/zero of=flash0.img bs=1M count=64
3549
dd if=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd of=flash0.img conv=notrunc
@@ -39,4 +53,4 @@ cd ..
3953
./packer init ./packer-scripts/arm-ubuntu.pkr.hcl
4054

4155
# Build the image with the specified Ubuntu version
42-
./packer build -var "ubuntu_version=${ubuntu_version}" ./packer-scripts/arm-ubuntu.pkr.hcl
56+
./packer build -var "ubuntu_version=${ubuntu_version}" -var "image_name=${image_name}" ./packer-scripts/arm-ubuntu.pkr.hcl

‎src/ubuntu-generic-diskimages/files/arm/after_boot.sh renamed to ‎src/ubuntu-generic-disk-images/files/arm/after_boot.sh

File renamed without changes.

‎src/ubuntu-generic-diskimages/files/arm/gem5_init.sh renamed to ‎src/ubuntu-generic-disk-images/files/arm/gem5_init.sh

+19
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,25 @@ mount -t sysfs /sys /sys
1515
cmdline=$(cat /proc/cmdline)
1616
no_systemd=false
1717

18+
# Load gem5_bridge driver
19+
## Default parameters (ARM64)
20+
gem5_bridge_baseaddr=0x10010000
21+
gem5_bridge_rangesize=0x10000
22+
## Try to read overloads from kernel arguments
23+
if [[ $cmdline =~ gem5_bridge_baseaddr=([[:alnum:]]+) ]]; then
24+
gem5_bridge_baseaddr=${BASH_REMATCH[1]}
25+
fi
26+
if [[ $cmdline =~ gem5_bridge_rangesize=([[:alnum:]]+) ]]; then
27+
gem5_bridge_rangesize=${BASH_REMATCH[1]}
28+
fi
29+
## Insert driver
30+
modprobe gem5_bridge \
31+
gem5_bridge_baseaddr=$gem5_bridge_baseaddr \
32+
gem5_bridge_rangesize=$gem5_bridge_rangesize
33+
34+
# see if this modprode fails or not
35+
# print warning if it fails, gem5-bridge module is not going to work, you will need sudo for running exit events
36+
1837
# gem5-bridge exit signifying that kernel is booted
1938
# This will cause the simulation to exit. Note that this will
2039
# cause qemu to fail.

‎src/ubuntu-generic-disk-images/files/[email protected]

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[Service]
2+
ExecStart=
3+
ExecStart=-/sbin/agetty --autologin gem5 --keep-baud 115200,38400,9600 %I $TERM

‎src/ubuntu-generic-diskimages/files/x86/after_boot.sh renamed to ‎src/ubuntu-generic-disk-images/files/x86/after_boot.sh

File renamed without changes.

‎src/ubuntu-generic-diskimages/files/x86/gem5_init.sh renamed to ‎src/ubuntu-generic-disk-images/files/x86/gem5_init.sh

File renamed without changes.

‎src/ubuntu-generic-diskimages/http/arm-22-04/meta-data renamed to ‎src/ubuntu-generic-disk-images/http/arm-2204/meta-data

File renamed without changes.

‎src/ubuntu-generic-diskimages/http/arm-22-04/user-data renamed to ‎src/ubuntu-generic-disk-images/http/arm-2204/user-data

+2-2
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ autoinstall:
7070
type: format
7171
id: format-0
7272
- device: disk-vda
73-
size: 4257218560
73+
size: 20909654016
7474
wipe: superblock
7575
flag: ''
7676
number: 2
@@ -92,4 +92,4 @@ autoinstall:
9292
type: mount
9393
id: mount-0
9494
updates: security
95-
version: 1
95+
version: 1

‎src/ubuntu-generic-diskimages/http/arm-24-04/meta-data renamed to ‎src/ubuntu-generic-disk-images/http/arm-2404/meta-data

File renamed without changes.

‎src/ubuntu-generic-diskimages/http/arm-24-04/user-data renamed to ‎src/ubuntu-generic-disk-images/http/arm-2404/user-data

+2-2
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ autoinstall:
6262
type: format
6363
id: format-0
6464
- device: disk-vda
65-
size: 4257218560
65+
size: 20909654016
6666
wipe: superblock
6767
flag: ''
6868
number: 2
@@ -84,4 +84,4 @@ autoinstall:
8484
type: mount
8585
id: mount-0
8686
updates: security
87-
version: 1
87+
version: 1

‎src/ubuntu-generic-diskimages/http/x86/meta-data renamed to ‎src/ubuntu-generic-disk-images/http/x86/meta-data

File renamed without changes.

‎src/ubuntu-generic-diskimages/http/x86/user-data renamed to ‎src/ubuntu-generic-disk-images/http/x86/user-data

File renamed without changes.

‎src/ubuntu-generic-disk-images/modules/u2204/Dockerfile

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
FROM ubuntu:22.04
2+
3+
# Install necessary packages for kernel build
4+
RUN apt update && apt install -y \
5+
build-essential \
6+
libncurses-dev \
7+
bison \
8+
flex \
9+
libssl-dev \
10+
libelf-dev \
11+
bc \
12+
wget \
13+
git \
14+
kmod \
15+
apt-src \
16+
vim \
17+
curl \
18+
file
19+
20+
RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
21+
RUN apt update
22+
RUN mkdir /workspace
23+
RUN cd /workspace && apt source linux-image-unsigned-5.15.0-25-generic
24+
25+
RUN cd /workspace/linux-5.15.0 && \
26+
cd scripts && \
27+
chmod +x pahole-version.sh && \
28+
cd .. && \
29+
make defconfig && \
30+
make -j 32 && \
31+
make INSTALL_MOD_PATH=/workspace/output modules_install
32+
33+
RUN git clone https://github.com/nkrim/gem5.git --depth=1 --filter=blob:none --no-checkout --sparse --single-branch --branch=gem5-bridge && \
34+
cd gem5 && \
35+
git sparse-checkout add util/m5 && \
36+
git sparse-checkout add util/gem5_bridge && \
37+
git sparse-checkout add include && \
38+
git checkout
39+
40+
RUN cd gem5/util/gem5_bridge && \
41+
make KMAKEDIR=/workspace/linux-5.15.0 INSTALL_MOD_PATH=/workspace/output build install
42+
43+
RUN cd /workspace/output/lib/modules/5.15.167 && \
44+
rm -rf build source
45+
46+
CMD ["/bin/bash"]

‎src/ubuntu-generic-disk-images/modules/u2204/copy_modules.sh

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Copyright (c) 2025 The Regents of the University of California.
4+
# SPDX-License-Identifier: BSD 3-Clause
5+
6+
# Variables
7+
IMAGE_NAME=ms-u2204-kernel-modules-source-image
8+
CONTAINER_NAME=ms-u2204-kernel-modules-source-container
9+
10+
# Check if the image exists
11+
if ! docker image inspect "$IMAGE_NAME" > /dev/null 2>&1; then
12+
echo "Docker image $IMAGE_NAME not found. Building the image..."
13+
14+
# Replace with the actual Docker build command
15+
docker build -t $IMAGE_NAME .
16+
17+
if [ $? -ne 0 ]; then
18+
echo "Failed to build the Docker image."
19+
exit 1
20+
fi
21+
else
22+
echo "Docker image $IMAGE_NAME exists."
23+
fi
24+
25+
# Check if the container already exists and stop/remove it if necessary
26+
if docker ps -a --format '{{.Names}}' | grep -q "^$CONTAINER_NAME\$"; then
27+
echo "Container $CONTAINER_NAME already exists. Removing it..."
28+
docker rm -f $CONTAINER_NAME
29+
fi
30+
31+
32+
# Create and start the container
33+
echo "Creating and starting container $CONTAINER_NAME..."
34+
docker create --name $CONTAINER_NAME $IMAGE_NAME
35+
docker start -a $CONTAINER_NAME
36+
37+
# Ensure the target directory exists
38+
mkdir -p files
39+
40+
# Copy files from the container to the host
41+
docker cp $CONTAINER_NAME:/workspace/linux-5.15.0/vmlinux files/
42+
docker cp $CONTAINER_NAME:/workspace/output/lib/modules/5.15.167 files/
43+
44+
# Stop and remove the container after the copy operation
45+
echo "Stopping and removing the container..."
46+
docker stop "$CONTAINER_NAME"
47+
docker rm "$CONTAINER_NAME"

‎src/ubuntu-generic-disk-images/modules/u2404/Dockerfile

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Start from Ubuntu 24.04 base image
2+
FROM ubuntu:24.04
3+
4+
# Install necessary packages for kernel and module build
5+
RUN apt update && apt install -y \
6+
build-essential \
7+
libncurses-dev \
8+
bison \
9+
flex \
10+
libssl-dev \
11+
libelf-dev \
12+
bc \
13+
wget \
14+
git \
15+
kmod
16+
17+
18+
RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
19+
RUN apt update
20+
RUN mkdir /workspace
21+
RUN cd /workspace && apt source linux-image-unsigned-6.8.0-47-generic
22+
23+
RUN cd /workspace/linux-6.8.0 && \
24+
make defconfig && \
25+
make -j 32 && \
26+
make INSTALL_MOD_PATH=/workspace/output modules_install
27+
28+
RUN git clone https://github.com/nkrim/gem5.git --depth=1 --filter=blob:none --no-checkout --sparse --single-branch --branch=gem5-bridge && \
29+
cd gem5 && \
30+
git sparse-checkout add util/m5 && \
31+
git sparse-checkout add util/gem5_bridge && \
32+
git sparse-checkout add include && \
33+
git checkout
34+
35+
RUN cd gem5/util/gem5_bridge && \
36+
make KMAKEDIR=/workspace/linux-6.8.0 INSTALL_MOD_PATH=/workspace/output build install
37+
38+
RUN cd /workspace/output/lib/modules/6.8.12 && \
39+
rm -rf build
40+
41+
# Use bash as the default shell
42+
CMD ["/bin/bash"]

‎src/ubuntu-generic-disk-images/modules/u2404/copy_modules.sh

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Copyright (c) 2025 The Regents of the University of California.
4+
# SPDX-License-Identifier: BSD 3-Clause
5+
6+
# Variables
7+
IMAGE_NAME=ms-u2404-kernel-modules-source-image
8+
CONTAINER_NAME=ms-u2404-kernel-modules-source-container
9+
10+
# Check if the image exists
11+
if ! docker image inspect "$IMAGE_NAME" > /dev/null 2>&1; then
12+
echo "Docker image $IMAGE_NAME not found. Building the image..."
13+
14+
# Replace with the actual Docker build command
15+
docker build -t $IMAGE_NAME .
16+
17+
if [ $? -ne 0 ]; then
18+
echo "Failed to build the Docker image."
19+
exit 1
20+
fi
21+
else
22+
echo "Docker image $IMAGE_NAME exists."
23+
fi
24+
25+
# Check if the container already exists and stop/remove it if necessary
26+
if docker ps -a --format '{{.Names}}' | grep -q "^$CONTAINER_NAME\$"; then
27+
echo "Container $CONTAINER_NAME already exists. Removing it..."
28+
docker rm -f $CONTAINER_NAME
29+
fi
30+
31+
32+
# Create and start the container
33+
echo "Creating and starting container $CONTAINER_NAME..."
34+
docker create --name $CONTAINER_NAME $IMAGE_NAME
35+
docker start -a $CONTAINER_NAME
36+
37+
# Ensure the target directory exists
38+
mkdir -p files
39+
40+
# Copy files from the container to the host
41+
docker cp $CONTAINER_NAME:/workspace/linux-6.8.0/vmlinux files/
42+
docker cp $CONTAINER_NAME:/workspace/output/lib/modules/6.8.12 files/
43+
44+
# Stop and remove the container after the copy operation
45+
echo "Stopping and removing the container..."
46+
docker stop "$CONTAINER_NAME"
47+
docker rm "$CONTAINER_NAME"

‎src/ubuntu-generic-diskimages/packer-scripts/arm-ubuntu.pkr.hcl renamed to ‎src/ubuntu-generic-disk-images/packer-scripts/arm-ubuntu.pkr.hcl

+31-17
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ packer {
99

1010
variable "image_name" {
1111
type = string
12-
default = "arm-ubuntu"
1312
}
1413

1514
variable "ssh_password" {
@@ -36,14 +35,18 @@ locals {
3635
"22.04" = {
3736
iso_url = "https://old-releases.ubuntu.com/releases/jammy/ubuntu-22.04-live-server-arm64.iso"
3837
iso_checksum = "sha256:c209ab013280d3cd26a344def60b7b19fbb427de904ea285057d94ca6ac82dd5"
39-
output_dir = "arm-disk-image-22-04"
40-
http_directory = "http/arm-22-04"
38+
output_dir = "${var.image_name}-2204"
39+
http_directory = "http/arm-2204"
40+
kernel_version = "5.15.167"
41+
modules_dir = "modules/u2204/files/"
4142
}
4243
"24.04" = {
43-
iso_url = "https://cdimage.ubuntu.com/releases/24.04/release/ubuntu-24.04-live-server-arm64.iso"
44-
iso_checksum = "sha256:d2d9986ada3864666e36a57634dfc97d17ad921fa44c56eeaca801e7dab08ad7"
45-
output_dir = "arm-disk-image-24-04"
46-
http_directory = "http/arm-24-04"
44+
iso_url = "https://cdimage.ubuntu.com/releases/24.04/release/ubuntu-24.04.1-live-server-arm64.iso"
45+
iso_checksum = "sha256:5ceecb7ef5f976e8ab3fffee7871518c8e9927ec221a3bb548ee1193989e1773"
46+
output_dir = "${var.image_name}-2404"
47+
http_directory = "http/arm-2404"
48+
kernel_version = "6.8.12"
49+
modules_dir = "modules/u2404/files/"
4750
}
4851
}
4952
}
@@ -91,22 +94,22 @@ source "qemu" "initialize" {
9194
"<enter>",
9295
"<wait>"
9396
]
94-
cpus = "4"
95-
disk_size = "4600"
97+
cpus = "32"
98+
disk_size = "21600"
9699
format = "raw"
97100
headless = "true"
98101
http_directory = local.iso_data[var.ubuntu_version].http_directory
99102
iso_checksum = local.iso_data[var.ubuntu_version].iso_checksum
100103
iso_urls = [local.iso_data[var.ubuntu_version].iso_url]
101-
memory = "8192"
104+
memory = "65536"
102105
output_directory = local.iso_data[var.ubuntu_version].output_dir
103106
qemu_binary = "/usr/bin/qemu-system-aarch64"
104107
qemuargs = local.qemuargs
105108
shutdown_command = "echo '${var.ssh_password}'|sudo -S shutdown -P now"
106109
ssh_password = "${var.ssh_password}"
107110
ssh_username = "${var.ssh_username}"
108111
ssh_wait_timeout = "60m"
109-
vm_name = "${var.image_name}"
112+
vm_name = "disk-image"
110113
ssh_handshake_attempts = "1000"
111114
}
112115

@@ -115,28 +118,39 @@ build {
115118

116119
provisioner "file" {
117120
destination = "/home/gem5/"
118-
source = "files/exit.sh"
121+
source = "files/arm/gem5_init.sh"
119122
}
120123

121124
provisioner "file" {
122125
destination = "/home/gem5/"
123-
source = "files/arm/gem5_init.sh"
126+
source = "files/arm/after_boot.sh"
124127
}
125128

126129
provisioner "file" {
127130
destination = "/home/gem5/"
128-
source = "files/arm/after_boot.sh"
131+
source = "files/[email protected]"
129132
}
130133

131134
provisioner "file" {
132135
destination = "/home/gem5/"
133-
source = "files/[email protected]"
136+
source = "${local.iso_data[var.ubuntu_version].modules_dir}/${local.iso_data[var.ubuntu_version].kernel_version}"
134137
}
135138

136139
provisioner "shell" {
137140
execute_command = "echo '${var.ssh_password}' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'"
138-
scripts = ["scripts/post-installation.sh"]
139-
environment_vars = ["ISA=arm64"]
141+
scripts = ["scripts/install-commons.sh", "scripts/update-modules.sh", "scripts/install-gem5-bridge-driver.sh", "scripts/install-user-packages.sh"]
142+
environment_vars = ["ISA=arm64", "KERNEL_VERSION=${local.iso_data[var.ubuntu_version].kernel_version}", "DEBIAN_FRONTEND=noninteractive"]
140143
expect_disconnect = true
141144
}
145+
146+
provisioner "shell" {
147+
scripts = ["scripts/install-benchmarks.sh"]
148+
}
149+
150+
provisioner "shell" {
151+
execute_command = "echo '${var.ssh_password}' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'"
152+
scripts = ["scripts/install-gem5-init.sh", "scripts/disable-network.sh"]
153+
expect_disconnect = true
154+
}
155+
142156
}

0 commit comments

Comments
 (0)