feature: Enable encryption x manual rollback flow

.pipelines/templates/stages/testing_common/e2e-test-run.yml

@@ -174,13 +174,13 @@ steps:
     condition: and(succeeded(), ne(variables['abActiveVolume'], 'null'))
 
   - ${{ if eq(parameters.deploymentEnvironment, 'virtualMachine') }}:
-    - bash: |
-        set -eux
-        ./bin/storm-trident script capture-screenshot \
-            --screenshot-filename "update-b.png" \
-            --artifacts-folder "$(ob_outputDirectory)"
-      displayName: "📷 Capture screenshot: stage and finalize A/B update into target OS B"
-      condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null'))
+      - bash: |
+          set -eux
+          ./bin/storm-trident script capture-screenshot \
+              --screenshot-filename "update-b.png" \
+              --artifacts-folder "$(ob_outputDirectory)"
+        displayName: "📷 Capture screenshot: stage and finalize A/B update into target OS B"
+        condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null'))
 
   - bash: |
       set -eux
@@ -324,13 +324,13 @@ steps:
     condition: and(succeeded(), ne(variables['abActiveVolume'], 'null'))
 
   - ${{ if eq(parameters.deploymentEnvironment, 'virtualMachine') }}:
-    - bash: |
-        set -eux
-        ./bin/storm-trident script capture-screenshot \
-            --screenshot-filename "update-a.png" \
-            --artifacts-folder "$(ob_outputDirectory)"
-      displayName: "📷 Capture screenshot: stage and finalize A/B update into target OS A"
-      condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null'))
+      - bash: |
+          set -eux
+          ./bin/storm-trident script capture-screenshot \
+              --screenshot-filename "update-a.png" \
+              --artifacts-folder "$(ob_outputDirectory)"
+        displayName: "📷 Capture screenshot: stage and finalize A/B update into target OS A"
+        condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null'))
 
   - bash: |
       set -eux
@@ -428,3 +428,75 @@ steps:
           --deployment-environment "${{ parameters.deploymentEnvironment }}"
     displayName: "Test Trident rebuild-raid"
     timeoutInMinutes: 15
+
+  # For 'combined' tests, trigger a rollback and test encryption
+  - bash: |
+      set -eux
+      if [ "${{ parameters.tridentConfigurationName }}" == "combined" ]; then
+        # If there is a netlisten process, kill it so there is no port clash in the instance
+        if pgrep netlisten > /dev/null; then pkill netlisten; fi
+        NETLISTEN_CONFIG_ARGS=""
+        if [ -n "${{ parameters.netlistenConfigFile }}" ]; then
+          NETLISTEN_CONFIG_ARGS="--config ${{ parameters.netlistenConfigFile }}"
+        fi
+        ./bin/netlisten --force-color $NETLISTEN_CONFIG_ARGS \
+            -m $(Build.SourcesDirectory)/trident-rollback-metrics.jsonl \
+            -p ${{ parameters.netlistenPort }} \
+            -s "${{ parameters.artifactsDirectory }}" > ./rollback.log 2>&1 &
+        PROXY_ARG=""
+        if [ -n "${{ parameters.httpsProxy }}" ]; then
+          PROXY_ARG="--env-vars HTTPS_PROXY=${{ parameters.httpsProxy }}"
+        fi
+        # To allow storm to access VM serial logs, need sudo.
+        sudo $(Build.SourcesDirectory)/bin/storm-trident helper manual-rollback -a \
+            --log-dir "$(ob_outputDirectory)" \
+            "${{ parameters.sshKeyPath }}" \
+            "${{ parameters.hostIp }}" \
+            "${{ parameters.userName }}" \
+            "${{ parameters.runtimeEnv }}" \
+            $PROXY_ARG
+      fi
+    displayName: "🔄 Stage and finalize manual rollback"
+    timeoutInMinutes: 5
+    condition: and(succeeded(), ne(variables['abActiveVolume'], 'null'))
+
+  - bash: |
+      set -eux
+      if [ "${{ parameters.tridentConfigurationName }}" == "combined" ]; then
+        # sudo is needed to read qemu VM serial logs, `-a` is needed with sudo
+        sudo ./bin/storm-trident helper display-logs -a \
+            --netlisten-config "${{ parameters.netlistenConfigFile }}" \
+            --serial-log-artifact-file-name "rollback-target-os-${TARGET_OS}-serial.log" \
+            --artifacts-folder "$(ob_outputDirectory)"
+      fi
+    env:
+      ${{ if or(eq(parameters.buildPurpose, 'daily'), eq(parameters.buildPurpose, 'validation'), eq(parameters.buildPurpose, 'weekly')) }}:
+        TARGET_OS: "A"
+      ${{ else }}:
+        TARGET_OS: "B"
+    displayName: "📄 Manual rollback deployment logs"
+    condition: and(succeededOrFailed(), ne(variables['abActiveVolume'], 'null'))
+
+  - bash: |
+      set -eux
+      if [ "${{ parameters.tridentConfigurationName }}" == "combined" ]; then
+        python3 -u -m pytest \
+          -m encryption \
+          --capture=no \
+          --junit-xml=${{ parameters.deploymentEnvironment }}_${{ parameters.runtimeEnv }}_${{ parameters.tridentConfigurationName }}_${{ parameters.jUnitXMLIdentifier }}_ab_rollback_${TARGET_OS}_$(System.JobAttempt).junit.xml \
+          --host ${{ parameters.hostIp }} \
+          --runtime-env ${{ parameters.runtimeEnv }} \
+          --configuration ${{ parameters.tridentConfigPath }} \
+          --ab-active-volume ${EXPECTED_VOLUME}
+      fi
+    env:
+      ${{ if or(eq(parameters.buildPurpose, 'daily'), eq(parameters.buildPurpose, 'validation'), eq(parameters.buildPurpose, 'weekly')) }}:
+        EXPECTED_VOLUME: "volume-a"
+        TARGET_OS: "A"
+      ${{ else }}:
+        EXPECTED_VOLUME: "volume-b"
+        TARGET_OS: "B"
+    timeoutInMinutes: 5
+    workingDirectory: $(Build.SourcesDirectory)/tests/e2e_tests
+    displayName: "🔬 Run Trident E2E tests after manual rollback"
+    condition: and(succeeded(), ne(variables['abActiveVolume'], 'null'))

crates/osutils/src/pcrlock.rs

@@ -182,9 +182,6 @@ fn generate_tpm2_access_policy(pcrs: BitFlags<Pcr>) -> Result<(), Error> {
 /// Please refer to `systemd-pcrlock` doc for additional info:
 /// https://www.man7.org/linux/man-pages/man8/systemd-pcrlock.8.html.
 fn validate_log(required_pcrs: BitFlags<Pcr>) -> Result<(), Error> {
-    // Print out combined TPM 2.0 event log
-    cel().context("Failed to print out combined TPM 2.0 event log")?;
-
     debug!(
         "Validating 'systemd-pcrlock log' output for required PCRs: {:?}",
         required_pcrs
@@ -259,16 +256,6 @@ fn log_parsed() -> Result<LogOutput, Error> {
     Ok(parsed_log)
 }
 
-/// Runs the `systemd-pcrlock cel` command to print out the combined TPM 2.0 event log in TCG
-/// Canonical Event Log Format (CEL-JSON).
-fn cel() -> Result<(), Error> {
-    Dependency::SystemdPcrlock
-        .cmd()
-        .arg("cel")
-        .run_and_check()
-        .context("Failed to run 'systemd-pcrlock cel'")
-}
-
 /// Returns a list of entries from the 'systemd-pcrlock log' output that correspond to (1) PCRs
 /// required for the pcrlock policy and (2) have components that are not recognized.
 fn unrecognized_log_entries(

crates/trident/src/engine/boot/uki.rs

@@ -197,9 +197,14 @@ fn enumerate_non_trident_managed_ukis(
     Ok(non_trident_managed_ukis)
 }
 
-/// Get list of UKI files (both Trident-managed and pre-existing) and find the previous UKI to use for rollback.
+/// Get list of UKI files (both Trident-managed and pre-existing) and find the previous UKI to use
+/// for rollback.
 pub fn find_previous_uki(esp_dir_path: &Path) -> Result<PathBuf, TridentError> {
     let esp_uki_directory = esp_dir_path.join(UKI_DIRECTORY);
+    debug!(
+        "Searching for previous UKI in directory '{}'",
+        esp_uki_directory.display()
+    );
     let trident_managed_ukis = enumerate_trident_managed_ukis(&esp_uki_directory)
         .structured(ServicingError::EnumerateUkis)
         .message("Failed to enumerate Trident-managed UKIs")?;

crates/trident/src/engine/manual_rollback/mod.rs

@@ -3,9 +3,10 @@ use std::{
     time::Instant,
 };
 
+use enumflags2::BitFlags;
 use log::{debug, info, trace, warn};
 
-use osutils::efivar;
+use osutils::{efivar, pcrlock};
 use trident_api::{
     config::Operations,
     constants::{
@@ -19,7 +20,10 @@ use crate::{
     cli::GetKind,
     container,
     datastore::DataStore,
-    engine::{self, boot::uki, bootentries, runtime_update, EngineContext, SUBSYSTEMS},
+    engine::{
+        self, boot::uki, bootentries, runtime_update, storage::encryption, EngineContext,
+        SUBSYSTEMS,
+    },
     subsystems::esp,
     ExitKind,
 };
@@ -225,32 +229,25 @@ fn stage_rollback(
     if matches!(staging_state, ServicingState::ManualRollbackAbStaged) {
         info!("Staging rollback of A/B update that requires reboot");
 
-        // If we have encrypted volumes and this is a UKI image, then we need to re-generate pcrlock
-        // policy to include both the current boot and the rollback boot.
-        if let Some(ref _encryption) = engine_context.spec.storage.encryption {
-            // TODO: We know how to update the pcrlock policy in the servicing OS, but are
-            // not able to do so for the target OS yet.
+        // If we have encrypted volumes and this is a UKI image, then we need to re-generate
+        // pcrlock policy to include both current boot and rollback boots.
+        if let Some(encryption) = &engine_context.spec.storage.encryption {
             if engine_context.is_uki()? {
-                return Err(TridentError::new(ServicingError::ManualRollback {
-                    message: "Cannot update pcrlock policy for UKI images during manual rollback",
-                }));
-                // debug!("Regenerating pcrlock policy to include rollback boot");
-
-                // // Get the PCRs from Host Configuration
-                // let pcrs = encryption
-                //     .pcrs
-                //     .iter()
-                //     .fold(BitFlags::empty(), |acc, &pcr| acc | BitFlags::from(pcr));
+                debug!("Regenerating pcrlock policy to include rollback boot");
 
-                // // Get UKI and bootloader binaries for .pcrlock file generation
-                // let (uki_binaries, bootloader_binaries) =
-                //     encryption::get_binary_paths_pcrlock(engine_context, pcrs, None, true)
-                //         .structured(ServicingError::GetBinaryPathsForPcrlockEncryption)?;
+                // Get the PCRs from Host Configuration
+                let pcrs = encryption
+                    .pcrs
+                    .iter()
+                    .fold(BitFlags::empty(), |acc, &pcr| acc | BitFlags::from(pcr));
 
-                // // Generate a pcrlock policy
-                // pcrlock::generate_pcrlock_policy(pcrs, uki_binaries, bootloader_binaries)?;
+                // Get UKI and bootloader binaries for .pcrlock file generation
+                let (uki_binaries, bootloader_binaries) =
+                    encryption::get_binary_paths_pcrlock(engine_context, pcrs, None, true)
+                        .structured(ServicingError::GetBinaryPathsForPcrlockEncryption)?;
 
-                // // Update the rollback OS pcrlock.json file
+                // Generate a pcrlock policy
+                pcrlock::generate_pcrlock_policy(pcrs, uki_binaries, bootloader_binaries)?;
             } else {
                 debug!(
                     "Rollback OS is a grub image, \