[SNP Guest VSM] Implement VTL Return hypercall handling (#224)

Implements VTL return hypercall handling for guest vsm support.

Tested:
- SNP VM boots
- SNP + VSM returns back to VTL 0

Author: sluck-msft

openhcl/virt_mshv_vtl/src/processor/hardware_cvm/mod.rs

@@ -19,9 +19,13 @@ use hvdef::HvError;
 use hvdef::HvMapGpaFlags;
 use hvdef::HvRegisterVsmPartitionConfig;
 use hvdef::HvResult;
+use hvdef::HvVtlEntryReason;
+use hvdef::HvX64RegisterName;
 use hvdef::Vtl;
 use std::iter::zip;
 use virt::io::CpuIo;
+use virt::vp::AccessVpState;
+use virt::Processor;
 use zerocopy::FromZeroes;
 
 impl<T: CpuIo, B: HardwareIsolatedBacking> UhHypercallHandler<'_, '_, T, B> {
@@ -200,7 +204,7 @@ impl<T: CpuIo, B: HardwareIsolatedBacking> UhHypercallHandler<'_, '_, T, B> {
         *target_vp.hv_start_enable_vtl_vp[vtl].lock() = Some(Box::new(*vp_context));
         target_vp.wake(vtl, WakeReason::HV_START_ENABLE_VP_VTL);
 
-        tracing::debug!("enabled vtl 1 on vp {}", vp_index);
+        tracing::debug!(vp_index, "enabled vtl 1 on vp");
 
         Ok(())
     }
@@ -238,18 +242,29 @@ impl<T: CpuIo, B: HardwareIsolatedBacking> UhHypercallHandler<'_, '_, T, B> {
         vtl: GuestVtl,
     ) -> HvResult<hvdef::HvRegisterValue> {
         match name.into() {
-            hvdef::HvX64RegisterName::VsmCodePageOffsets => Ok(u64::from(
+            HvX64RegisterName::VsmCodePageOffsets => Ok(u64::from(
                 self.vp.hv[vtl]
                     .as_ref()
                     .expect("hv emulator exists for cvm")
                     .vsm_code_page_offsets(true),
             )
             .into()),
-            hvdef::HvX64RegisterName::VsmCapabilities => Ok(u64::from(
+            HvX64RegisterName::VsmCapabilities => Ok(u64::from(
                 hvdef::HvRegisterVsmCapabilities::new().with_deny_lower_vtl_startup(true),
             )
             .into()),
-            _ => Err(HvError::InvalidParameter),
+            HvX64RegisterName::VpAssistPage => Ok(self.vp.hv[vtl]
+                .as_ref()
+                .expect("hv emulator exists for cvm")
+                .vp_assist_page()
+                .into()),
+            _ => {
+                tracing::error!(
+                    ?name,
+                    "guest invoked getvpregister with unsupported register"
+                );
+                Err(HvError::InvalidParameter)
+            }
         }
     }
 
@@ -329,32 +344,6 @@ impl<T: CpuIo, B: HardwareIsolatedBacking> UhHypercallHandler<'_, '_, T, B> {
         // TODO should we check the all_virtual_address_spaces flag? we don't check this flag or the address space input arg anywhere in the hcl
         Ok(())
     }
-
-    pub fn hcvm_is_vtl_call_allowed(&self) -> bool {
-        tracing::trace!("checking if vtl call is allowed");
-
-        // Only allowed from VTL 0
-        if self.intercepted_vtl != GuestVtl::Vtl0 {
-            false
-        } else if !*self.vp.inner.hcvm_vtl1_enabled.lock() {
-            // VTL 1 must be enabled on the vp
-            false
-        } else {
-            true
-        }
-    }
-
-    pub fn hcvm_vtl_call(&mut self) {
-        tracing::trace!("handling vtl call");
-
-        self.vp.switch_vtl(self.intercepted_vtl, GuestVtl::Vtl1);
-        self.vp.backing.cvm_state_mut().exit_vtl = GuestVtl::Vtl1;
-
-        // TODO GUEST_VSM: Force reevaluation of the VTL 1 APIC in case delivery of
-        // low-priority interrupts was suppressed while in VTL 0.
-
-        // TODO GUEST_VSM: Track which VTLs are runnable and mark VTL as runnable
-    }
 }
 
 impl<T, B: HardwareIsolatedBacking> hv1_hypercall::SetVpRegisters
@@ -380,20 +369,112 @@ impl<T, B: HardwareIsolatedBacking> hv1_hypercall::SetVpRegisters
             .map_err(|_| (HvError::InvalidParameter, 0))?;
 
         for (i, reg) in registers.iter().enumerate() {
-            if reg.name == hvdef::HvX64RegisterName::VsmPartitionConfig.into() {
-                let value = HvRegisterVsmPartitionConfig::from(reg.value.as_u64());
-                self.vp
-                    .set_vsm_partition_config(value, target_vtl)
-                    .map_err(|e| (e, i))?;
-            } else {
-                return Err((HvError::InvalidParameter, i));
+            match HvX64RegisterName::from(reg.name) {
+                HvX64RegisterName::VsmPartitionConfig => {
+                    self.vp
+                        .set_vsm_partition_config(
+                            HvRegisterVsmPartitionConfig::from(reg.value.as_u64()),
+                            target_vtl,
+                        )
+                        .map_err(|e| (e, i))?;
+                }
+                HvX64RegisterName::VpAssistPage => {
+                    self.vp.hv[target_vtl]
+                        .as_mut()
+                        .expect("has hv emulator")
+                        .msr_write(hvdef::HV_X64_MSR_VP_ASSIST_PAGE, reg.value.as_u64())
+                        .map_err(|_| (HvError::InvalidRegisterValue, 0))?;
+                }
+                _ => {
+                    tracing::error!(
+                        ?reg,
+                        "guest invoked SetVpRegisters with unsupported register"
+                    );
+                    return Err((HvError::InvalidParameter, i));
+                }
             }
         }
 
         Ok(())
     }
 }
 
+impl<T, B: HardwareIsolatedBacking> hv1_hypercall::VtlCall for UhHypercallHandler<'_, '_, T, B> {
+    fn is_vtl_call_allowed(&self) -> bool {
+        tracing::trace!("checking if vtl call is allowed");
+
+        // Only allowed from VTL 0
+        if self.intercepted_vtl != GuestVtl::Vtl0 {
+            false
+        } else if !*self.vp.inner.hcvm_vtl1_enabled.lock() {
+            // VTL 1 must be enabled on the vp
+            false
+        } else {
+            true
+        }
+    }
+
+    fn vtl_call(&mut self) {
+        tracing::trace!("handling vtl call");
+
+        B::switch_vtl_state(self.vp, self.intercepted_vtl, GuestVtl::Vtl1);
+        self.vp.backing.cvm_state_mut().exit_vtl = GuestVtl::Vtl1;
+
+        // TODO GUEST VSM: reevaluate if the return reason should be set here or
+        // during VTL 2 exit handling
+        self.vp.hv[GuestVtl::Vtl1]
+            .as_ref()
+            .unwrap()
+            .set_return_reason(HvVtlEntryReason::VTL_CALL)
+            .expect("setting return reason cannot fail");
+
+        // TODO GUEST_VSM: Force reevaluation of the VTL 1 APIC in case delivery of
+        // low-priority interrupts was suppressed while in VTL 0.
+
+        // TODO GUEST_VSM: Track which VTLs are runnable and mark VTL as runnable
+    }
+}
+
+impl<T, B: HardwareIsolatedBacking> hv1_hypercall::VtlReturn for UhHypercallHandler<'_, '_, T, B> {
+    fn is_vtl_return_allowed(&self) -> bool {
+        tracing::trace!("checking if vtl return is allowed");
+
+        // Only allowed from VTL 1
+        self.intercepted_vtl != GuestVtl::Vtl0
+    }
+
+    fn vtl_return(&mut self, fast: bool) {
+        tracing::trace!("handling vtl return");
+
+        self.vp.unlock_tlb_lock(Vtl::Vtl1);
+
+        B::switch_vtl_state(self.vp, self.intercepted_vtl, GuestVtl::Vtl0);
+        self.vp.backing.cvm_state_mut().exit_vtl = GuestVtl::Vtl0;
+
+        // TODO CVM GUEST_VSM:
+        // - rewind interrupts
+        // - reset VINA
+
+        if !fast {
+            let [rax, rcx] = self.vp.hv[GuestVtl::Vtl1]
+                .as_ref()
+                .unwrap()
+                .return_registers()
+                .expect("getting return registers shouldn't fail");
+            let mut vp_state = self.vp.access_state(Vtl::Vtl0);
+            let mut registers = vp_state
+                .registers()
+                .expect("getting registers shouldn't fail");
+            registers.rax = rax;
+            registers.rcx = rcx;
+
+            vp_state
+                .set_registers(&registers)
+                .expect("setting registers shouldn't fail");
+        }
+    }
+}
+
 impl<B: HardwareIsolatedBacking> UhProcessor<'_, B> {
     fn set_vsm_partition_config(
         &mut self,

openhcl/virt_mshv_vtl/src/processor/mod.rs

@@ -1112,11 +1112,6 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
 
         self.request_sint_notifications(vtl, pending_sints);
     }
-
-    #[cfg_attr(guest_arch = "aarch64", allow(dead_code))]
-    fn switch_vtl(&mut self, source_vtl: GuestVtl, target_vtl: GuestVtl) {
-        T::switch_vtl_state(self, source_vtl, target_vtl);
-    }
 }
 
 fn signal_mnf(dev: &impl CpuIo, connection_id: u32) {

openhcl/virt_mshv_vtl/src/processor/snp/mod.rs

@@ -634,6 +634,7 @@ impl<T: CpuIo> UhHypercallHandler<'_, '_, T, SnpBacked> {
             hv1_hypercall::HvX64EnableVpVtl,
             hv1_hypercall::HvExtQueryCapabilities,
             hv1_hypercall::HvVtlCall,
+            hv1_hypercall::HvVtlReturn,
             hv1_hypercall::HvFlushVirtualAddressList,
             hv1_hypercall::HvFlushVirtualAddressListEx,
             hv1_hypercall::HvFlushVirtualAddressSpace,
@@ -2212,16 +2213,6 @@ impl<T: CpuIo> hv1_hypercall::RetargetDeviceInterrupt for UhHypercallHandler<'_,
     }
 }
 
-impl<T: CpuIo> hv1_hypercall::VtlCall for UhHypercallHandler<'_, '_, T, SnpBacked> {
-    fn is_vtl_call_allowed(&self) -> bool {
-        self.hcvm_is_vtl_call_allowed()
-    }
-
-    fn vtl_call(&mut self) {
-        self.hcvm_vtl_call()
-    }
-}
-
 impl<T: CpuIo> hv1_hypercall::VtlSwitchOps for UhHypercallHandler<'_, '_, T, SnpBacked> {
     fn advance_ip(&mut self) {
         let is_64bit = self.vp.long_mode(self.intercepted_vtl);

vm/hv1/hv1_emulator/src/hv.rs

@@ -10,6 +10,8 @@ use super::synic::ProcessorSynic;
 use guestmem::GuestMemory;
 use guestmem::GuestMemoryError;
 use hvdef::HvRegisterVpAssistPage;
+use hvdef::HvVpVtlControl;
+use hvdef::HvVtlEntryReason;
 use hvdef::HV_PAGE_SIZE;
 use hvdef::HV_PAGE_SIZE_USIZE;
 use hvdef::HV_REFERENCE_TSC_SEQUENCE_INVALID;
@@ -456,6 +458,25 @@ impl ProcessorVtlHv {
             false
         }
     }
+
+    /// Get the register values to restore on vtl return
+    pub fn return_registers(&self) -> Result<[u64; 2], GuestMemoryError> {
+        let gpa = (self.vp_assist_page.gpa_page_number() * HV_PAGE_SIZE)
+            + offset_of!(hvdef::HvVpAssistPage, vtl_control) as u64;
+
+        let v: HvVpVtlControl = self.guest_memory.read_plain(gpa)?;
+
+        Ok(v.registers)
+    }
+
+    /// Set the reason for the vtl return into the vp assist page
+    pub fn set_return_reason(&self, reason: HvVtlEntryReason) -> Result<(), GuestMemoryError> {
+        let gpa = (self.vp_assist_page.gpa_page_number() * HV_PAGE_SIZE)
+            + offset_of!(hvdef::HvVpAssistPage, vtl_control) as u64
+            + offset_of!(HvVpVtlControl, entry_reason) as u64;
+
+        self.guest_memory.write_plain(gpa, &(reason.0))
+    }
 }
 
 struct HypercallPage {