Install kernel packages before pinning

1. Move DisableNouveau, InstallAdditionalKernelPackages from `update_and_reboot.yaml` to `parallelcluster.yaml`. This improves consistency of build between with and without `UpdateOsPackages`
2. Remove InstallEfiBootManager because the bug it was trying to work around already got resolved.
3. Move RHEL minor version pinning from `InstallPrerequisite` to the beginning of `PinVersion` for the following reasons
3.1 Even if we make mistakes pinning packages without installing, with minor version pinned, we will pin to the latest within the minor
3.2 `yum install -y yum-plugin-versionlock` will install the version within the minor
3.3 In terms of code style, RHEL minor version pinning fits in `PinVersion` better than `InstallPrerequisite`
4. For RHEL/Rocky, install kernel-headers, kernel-devel, kernel-devel-matched(RHEL/Rocky >=9) before pinning the version. For Ubuntu, install `linux-headers` before pinning the version
5. Add a Reboot after installing and pinning kernel packages

Author: hanwen-cluster

CHANGELOG.md

@@ -9,6 +9,7 @@ CHANGELOG
 
 **BUG FIXES**
 - Fix an issue where Security Group validation failed when a rule contained both IPv4 ranges (IpRanges) and security group references (UserIdGroupPairs).
+- Fix build image failures on Rocky 9 non-lastest versions.
 
 3.13.1
 ------

cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml

@@ -182,7 +182,7 @@ phases:
                 fi
               fi
 
-      - name: PinKernelVersion
+      - name: PinVersion
         action: ExecuteBash
         inputs:
           commands:
@@ -191,7 +191,45 @@ phases:
               OS='{{ build.OperatingSystemName.outputs.stdout }}'
               PLATFORM='{{ build.PlatformName.outputs.stdout }}'
               KERNEL_VERSION=$(uname -a)
+              RELEASE_VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}'
               if [[ ${!PLATFORM} == RHEL ]]; then
+                if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then
+                  if [[ ! -f /etc/yum/vars/releasever ]]; then
+                    echo "yes" > /opt/parallelcluster/pin_releasesever
+                    echo ${!RELEASE_VERSION} > /etc/yum/vars/releasever
+                    yum clean all
+                  fi
+                fi
+                PACKAGE_LIST="kernel-headers-$(uname -r) kernel-devel-$(uname -r)"
+                if [[ ${!OS} != "rocky8" ]] && [[ ${!OS} != "rhel8" ]]; then
+                  PACKAGE_LIST+=" kernel-devel-matched-$(uname -r)"
+                fi
+
+                if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then
+                  if [[ ${!OS} == "rocky8" ]] ; then
+                    REPOSITORY="BaseOS"
+                  else
+                    REPOSITORY="AppStream"
+                  fi
+                  for PACKAGE in ${!PACKAGE_LIST}
+                  do
+                    # try to install kernel source for a specific release version
+                    yum install -y ${!PACKAGE}
+                    if [ $? -ne 0 ]; then
+                      yum install -y wget
+                      # Previous releases are moved into a vault area once a new minor release version is available for at least a week.
+                      # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel
+                      wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/${!REPOSITORY}/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm
+                      yum install -y ./${!PACKAGE}.rpm
+                    fi
+                  done
+                else
+                  for PACKAGE in ${!PACKAGE_LIST}
+                  do
+                    yum -y install ${!PACKAGE}
+                  done
+                fi
+
                 yum install -y yum-plugin-versionlock
                 # listing all the packages because wildcard does not work as expected
                 yum versionlock kernel kernel-core kernel-modules
@@ -202,9 +240,24 @@ phases:
                   yum versionlock redhat-release
                 fi
               else
+                apt-get -y install linux-headers-$(uname -r)
                 apt-mark hold linux-aws* linux-base* linux-headers* linux-image* 
               fi
-              echo "Kernel version is ${!KERNEL_VERSION}"     
+              echo "Kernel version is ${!KERNEL_VERSION}"
+
+      - name: DisableNouveau
+        action: ExecuteBash
+        inputs:
+          commands:
+            - |
+              set -v
+              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
+              /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
+              if [[ ${!PLATFORM} == RHEL ]]; then
+                grub2-mkconfig -o /boot/grub2/grub.cfg
+              elif [[ ${!PLATFORM} == DEBIAN ]]; then
+                update-grub
+              fi
 
       # Install prerequisite OS packages
       - name: InstallPrerequisite
@@ -215,16 +268,8 @@ phases:
               set -v
               OS='{{ build.OperatingSystemName.outputs.stdout }}'
               PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}'
 
               if [[ ${!PLATFORM} == RHEL ]]; then
-                if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then
-                  if [[ ! -f /etc/yum/vars/releasever ]]; then
-                    echo "yes" > /opt/parallelcluster/pin_releasesever
-                    echo ${!VERSION} > /etc/yum/vars/releasever
-                    yum clean all
-                  fi
-                fi
                 yum -y update krb5-libs
                 yum -y groupinstall development && sudo yum -y install wget jq
                 if [[ ${!OS} != alinux2023 ]]; then
@@ -244,6 +289,13 @@ phases:
                 apt-get -y install build-essential curl wget jq
               fi
 
+      - name: RebootStep
+        action: Reboot
+        onFailure: Abort
+        maxAttempts: 2
+        inputs:
+          delaySeconds: 10
+
       # Install Cinc
       - name: InstallCinc
         action: ExecuteBash

cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml

@@ -97,20 +97,6 @@ phases:
                 fi
               fi
 
-      - name: DisableNouveau
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
-              if [[ ${!PLATFORM} == RHEL ]]; then
-                grub2-mkconfig -o /boot/grub2/grub.cfg
-              elif [[ ${!PLATFORM} == DEBIAN ]]; then
-                update-grub
-              fi
-
       - name: DisableUnattendedUpgrades
         action: ExecuteBash
         inputs:
@@ -129,20 +115,6 @@ phases:
                 # update package index
                 DEBIAN_FRONTEND=noninteractive apt-get -y update
               fi
-
-      - name: InstallEfiBootManager
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              ARCH=$(uname -m)
-              if [[ `echo ${!ARCH}` == 'aarch64' ]] && [[ ${!PLATFORM} == DEBIAN ]]; then
-                # temporary workaround to solve https://bugs.launchpad.net/ubuntu/+source/grub2-signed/+bug/1936857
-                apt-get -y install efibootmgr
-              fi
-
       - name: InstallPrerequisites
         action: ExecuteBash
         inputs:
@@ -244,42 +216,6 @@ phases:
         maxAttempts: 2
         inputs:
           delaySeconds: 10
-      - name: InstallAdditionalKernelPackages
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              OS='{{ build.OperatingSystemName.outputs.stdout }}'
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              DISABLE_KERNEL_UPDATE='{{ build.DisableKernelUpdate.outputs.stdout }}'
-
-              if [[ ${!PLATFORM} == RHEL ]]; then
-                # Install kernel-devel during OS update, so that headers are aligned with new kernel.
-                # The same is done for Debian through `apt-get -y install linux-aws`
-                if [[ ${!OS} == "rocky8" ]] ; then
-                  PACKAGE="kernel-devel-$(uname -r)"
-                  RELEASE_VERSION=$(source /etc/os-release && echo ${!VERSION_ID})
-
-                  # try to install kernel source for a specific release version
-                  yum install -y ${!PACKAGE} --releasever ${!RELEASE_VERSION}
-                  if [ $? -ne 0 ]; then
-                    yum install -y wget
-                    # Previous releases are moved into a vault area once a new minor release version is available for at least a week.
-                    # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel
-                    wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/BaseOS/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm
-                    yum install -y ./${!PACKAGE}.rpm
-                  fi
-                else
-                  yum -y install kernel-headers-$(uname -r)
-                  yum -y install kernel-devel-$(uname -r)
-                fi
-
-              elif [[ ${!PLATFORM} == DEBIAN ]]; then
-                if [[ ${!DISABLE_KERNEL_UPDATE} != true ]]; then
-                   apt-get -y install linux-aws linux-headers-aws linux-image-aws
-                fi
-              fi
 
       - name: RemoveKernelPin
         action: ExecuteBash