feat(slurmd): add SSH support with pam_slurm_adopt

Enable SSH access to slurmd containers with job-based access control via
pam_slurm_adopt. SSH host keys are generated at runtime, and PAM is
dynamically configured to restrict SSH access to users with active jobs

Ref: https://slurm.schedmd.com/pam_slurm_adopt.html

Author: faganihajizada

schedmd/slurm/25.11/rockylinux9/Dockerfile

@@ -183,6 +183,7 @@ RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked <<EOR
 set -xeuo pipefail
 dnf -q -y install --setopt='install_weak_deps=False' \
   gawk socat \
+  openssh-server \
   ./slurm-devel-[0-9]*.rpm \
   ./slurm-libpmi-[0-9]*.rpm \
   ./slurm-pam_slurm-[0-9]*.rpm \
@@ -194,17 +195,21 @@ mkdir -p /var/spool/slurmd/
 cp -v /etc/nsswitch.conf{,.bak}
 sed -i -E "s/^passwd:[[:space:]]+/&slurm /g" /etc/nsswitch.conf
 sed -i -E "s/^group:[[:space:]]+/&slurm /g" /etc/nsswitch.conf
+# Configure SSH
+rm -f /etc/ssh/ssh_host_*
 EOR
 
 COPY files/etc/supervisord.conf /etc/
 COPY \
   files/etc/supervisord.d/slurmd.ini \
   files/etc/supervisord.d/fakesystemd.ini \
+  files/etc/supervisord.d/sshd.ini \
   /etc/supervisord.d/
 COPY files/usr/local/bin/fakesystemd.sh /usr/local/bin/
 COPY files/usr/local/bin/slurmd-entrypoint.sh /usr/local/bin/entrypoint.sh
 
 EXPOSE 6818/tcp
+EXPOSE 22/tcp
 ENTRYPOINT ["entrypoint.sh"]
 
 ################################################################################

schedmd/slurm/25.11/rockylinux9/files/usr/local/bin/slurmd-entrypoint.sh

@@ -7,6 +7,9 @@ set -euo pipefail
 # Additional arguments to pass to slurmd.
 export SLURMD_OPTIONS="${SLURMD_OPTIONS:-} $*"
 
+# Additional arguments to pass to sshd.
+export SSHD_OPTIONS="${SSHD_OPTIONS:-""}"
+
 # The asserted CPU resource limit of the pod.
 export POD_CPUS="${POD_CPUS:-0}"
 
@@ -94,6 +97,17 @@ function addConfItem() {
 	export SLURMD_OPTIONS="${slurmdOptions[*]}"
 }
 
+# configure_pam configures PAM to use pam_slurm_adopt for SSH sessions.
+#
+# This allows SSH access to be restricted to users with active jobs on the node.
+function configure_pam() {
+	# Add pam_slurm_adopt to SSH PAM configuration if not already present
+	if ! grep -q "pam_slurm_adopt.so" /etc/pam.d/sshd 2>/dev/null; then
+		# Insert after account include password-auth (Rocky Linux specific pattern)
+		sed -i '/^account[[:space:]]*include[[:space:]]*password-auth/a -account   required     pam_slurm_adopt.so action_no_jobs=deny action_unknown=newest action_adopt_failure=deny action_generic_failure=deny disable_x11=0' /etc/pam.d/sshd
+	fi
+}
+
 function main() {
 	mkdir -p /run/slurm/
 	mkdir -p /var/spool/slurmd/
@@ -114,6 +128,12 @@ function main() {
 		addConfItem "MemSpecLimit=${memSpecLimit}"
 	fi
 
+	# Initialize SSH
+	mkdir -p /run/sshd/
+	chmod 0755 /run/sshd/
+	ssh-keygen -A
+	configure_pam
+
 	exec supervisord -c /etc/supervisord.conf
 }
 main

schedmd/slurm/25.11/ubuntu24.04/Dockerfile

@@ -179,6 +179,7 @@ set -xeuo pipefail
 apt-get -qq update
 apt-get -qq -y install --no-install-recommends --fix-broken \
   gawk socat \
+  openssh-server \
   ./slurm-smd-client_[0-9]*.deb \
   ./slurm-smd-client-dbgsym_[0-9]*.ddeb \
   ./slurm-smd-dev_[0-9]*.deb \
@@ -197,17 +198,21 @@ mkdir -p /var/spool/slurmd/
 cp -v /etc/nsswitch.conf{,.bak}
 sed -i -E "s/^passwd:[[:space:]]+/&slurm /g" /etc/nsswitch.conf
 sed -i -E "s/^group:[[:space:]]+/&slurm /g" /etc/nsswitch.conf
+# Configure SSH
+rm -f /etc/ssh/ssh_host_*
 EOR
 
 COPY files/etc/supervisor/supervisord.conf /etc/supervisor/
 COPY \
   files/etc/supervisor/conf.d/slurmd.conf \
   files/etc/supervisor/conf.d/fakesystemd.conf \
+  files/etc/supervisor/conf.d/sshd.conf \
   /etc/supervisor/conf.d/
 COPY files/usr/local/bin/fakesystemd.sh /usr/local/bin/
 COPY files/usr/local/bin/slurmd-entrypoint.sh /usr/local/bin/entrypoint.sh
 
 EXPOSE 6818/tcp
+EXPOSE 22/tcp
 ENTRYPOINT ["entrypoint.sh"]
 
 ################################################################################

schedmd/slurm/25.11/ubuntu24.04/files/usr/local/bin/slurmd-entrypoint.sh

@@ -7,6 +7,9 @@ set -euo pipefail
 # Additional arguments to pass to slurmd.
 export SLURMD_OPTIONS="${SLURMD_OPTIONS:-} $*"
 
+# Additional arguments to pass to sshd.
+export SSHD_OPTIONS="${SSHD_OPTIONS:-""}"
+
 # The asserted CPU resource limit of the pod.
 export POD_CPUS="${POD_CPUS:-0}"
 
@@ -94,6 +97,17 @@ function addConfItem() {
 	export SLURMD_OPTIONS="${slurmdOptions[*]}"
 }
 
+# configure_pam configures PAM to use pam_slurm_adopt for SSH sessions.
+#
+# This allows SSH access to be restricted to users with active jobs on the node.
+function configure_pam() {
+	# Add pam_slurm_adopt to SSH PAM configuration if not already present
+	if ! grep -q "pam_slurm_adopt.so" /etc/pam.d/sshd 2>/dev/null; then
+		# Insert after common-account include
+		sed -i '/^@include common-account/a -account   required     pam_slurm_adopt.so action_no_jobs=deny action_unknown=newest action_adopt_failure=deny action_generic_failure=deny disable_x11=0' /etc/pam.d/sshd
+	fi
+}
+
 function main() {
 	mkdir -p /run/slurm/
 	mkdir -p /var/spool/slurmd/
@@ -114,6 +128,12 @@ function main() {
 		addConfItem "MemSpecLimit=${memSpecLimit}"
 	fi
 
+	# Initialize SSH
+	mkdir -p /run/sshd/
+	chmod 0755 /run/sshd/
+	ssh-keygen -A
+	configure_pam
+
 	exec supervisord -c /etc/supervisor/supervisord.conf
 }
 main

schedmd/slurm/master/rockylinux9/files/usr/local/bin/slurmd-entrypoint.sh

@@ -95,11 +95,13 @@ function addConfItem() {
 	export SLURMD_OPTIONS="${slurmdOptions[*]}"
 }
 
-# Configure PAM for pam_slurm_adopt (following login's dynamic pattern)
+# configure_pam configures PAM to use pam_slurm_adopt for SSH sessions.
+#
+# This allows SSH access to be restricted to users with active jobs on the node.
 function configure_pam() {
 	# Add pam_slurm_adopt to SSH PAM configuration if not already present
 	if ! grep -q "pam_slurm_adopt.so" /etc/pam.d/sshd 2>/dev/null; then
-		# Insert after common-account include
+		# Insert after account include password-auth (Rocky Linux specific pattern)
 		sed -i '/^account[[:space:]]*include[[:space:]]*password-auth/a -account   required     pam_slurm_adopt.so action_no_jobs=deny action_unknown=newest action_adopt_failure=deny action_generic_failure=deny disable_x11=0' /etc/pam.d/sshd
 	fi
 }

schedmd/slurm/master/ubuntu24.04/files/usr/local/bin/slurmd-entrypoint.sh

@@ -95,7 +95,9 @@ function addConfItem() {
 	export SLURMD_OPTIONS="${slurmdOptions[*]}"
 }
 
-# Configure PAM for pam_slurm_adopt (following login's dynamic pattern)
+# configure_pam configures PAM to use pam_slurm_adopt for SSH sessions.
+#
+# This allows SSH access to be restricted to users with active jobs on the node.
 function configure_pam() {
 	# Add pam_slurm_adopt to SSH PAM configuration if not already present
 	if ! grep -q "pam_slurm_adopt.so" /etc/pam.d/sshd 2>/dev/null; then