snap/restore: save MSRs based on CPUID

Some additional MSRs need to be saved if certain features are enabled in
CPUID.
Add the capability to infer which MSRs to save based on the feature bits
set in CPUID. The dependencies between CPUID features and MSRs are
defined for each CPU vendor.
Through this commit, the list of MSRs we need to save becomes
architectural_msrs + msrs_based_on_cpuid + msrs_defined_in_template.

The dependency list now contains only one entry for the MPX feature bit
so it is by no means exhaustive. We'll add more dependencies in
following commits. Until then the right coverage should be supplied
through templates and the ALLOWED_MSR_RANGES list.

Signed-off-by: Alexandru-Cezar Sardan <[email protected]>

Author: Alexandru-Cezar Sardan

Cargo.lock

@@ -13,3 +13,4 @@ thiserror = "1.0.32"
 
 utils = { path = "../utils"}
 arch = { path = "../arch" }
+arch_gen = { path = "../arch_gen" }

src/cpuid/Cargo.toml

@@ -159,6 +159,25 @@ pub fn is_same_model(cpuid: &CpuId) -> bool {
     true
 }
 
+/// Scans through the CPUID and determines if a feature bit is set.
+// TODO: This currently involves a linear search which would be improved
+//       when we'll refactor the cpuid crate.
+#[macro_export]
+macro_rules! cpuid_is_feature_set {
+    ($cpuid:ident, $leaf:expr, $index:expr, $reg:tt, $feature_bit:expr) => {{
+        let mut res = false;
+        for entry in $cpuid.as_slice().iter() {
+            if entry.function == $leaf && entry.index == $index {
+                if entry.$reg & (1 << $feature_bit) != 0 {
+                    res = true;
+                    break;
+                }
+            }
+        }
+        res
+    }};
+}
+
 #[cfg(test)]
 pub mod tests {
     use crate::common::*;

src/cpuid/src/common.rs

@@ -20,6 +20,7 @@ pub mod bit_helper;
 
 mod template;
 pub use crate::template::intel::{c3, t2, t2s};
+pub use crate::template::msrs_to_save_by_cpuid;
 
 mod cpu_leaf;
 

src/cpuid/src/lib.rs

@@ -9,7 +9,13 @@ pub mod t2;
 /// that are speciffic to an Intel Skylake CPU.
 pub mod t2s;
 
+use std::collections::HashSet;
+
+use arch_gen::x86::msr_index::*;
+use kvm_bindings::CpuId;
+
 use crate::common::{get_vendor_id_from_host, VENDOR_ID_INTEL};
+use crate::cpuid_is_feature_set;
 use crate::transformer::Error;
 
 pub fn validate_vendor_id() -> Result<(), Error> {
@@ -20,3 +26,29 @@ pub fn validate_vendor_id() -> Result<(), Error> {
 
     Ok(())
 }
+
+/// Returns MSRs to be saved based on the Intel CPUID features that are enabled.
+pub(crate) fn msrs_to_save_by_cpuid(cpuid: &CpuId) -> HashSet<u32> {
+    use crate::cpu_leaf::*;
+
+    let mut msrs = HashSet::new();
+
+    // Macro used for easy definition of CPUID-MSR dependencies.
+    macro_rules! cpuid_msr_dep {
+        ($leaf:expr, $index:expr, $reg:tt, $feature_bit:expr, [$($msr:expr),+]) => {
+            if cpuid_is_feature_set!(cpuid, $leaf, $index, $reg, $feature_bit) {
+                msrs.extend(&[$($msr),*]);
+            }
+        };
+    }
+
+    // TODO: Add more dependencies.
+    cpuid_msr_dep!(
+        0x7,
+        0,
+        ebx,
+        leaf_0x7::index0::ebx::MPX_BITINDEX,
+        [MSR_IA32_BNDCFGS]
+    );
+    msrs
+}

src/cpuid/src/template/intel/mod.rs

@@ -3,3 +3,22 @@
 
 // Contains Intel specific templates.
 pub mod intel;
+
+use std::collections::HashSet;
+
+use kvm_bindings::CpuId;
+
+use crate::common::{get_vendor_id_from_cpuid, VENDOR_ID_INTEL};
+use crate::transformer::Error;
+
+/// Returns MSRs to be saved based on CPUID features that are enabled.
+pub fn msrs_to_save_by_cpuid(cpuid: &CpuId) -> Result<HashSet<u32>, Error> {
+    let vendor_id = get_vendor_id_from_cpuid(cpuid).map_err(|_| Error::InvalidVendor)?;
+    match &vendor_id {
+        VENDOR_ID_INTEL => Ok(intel::msrs_to_save_by_cpuid(cpuid)),
+        _ => {
+            // We don't have MSR-CPUID dependencies set for other vendors yet.
+            Ok(HashSet::new())
+        }
+    }
+}

src/cpuid/src/template/mod.rs

@@ -11,7 +11,7 @@ use std::{fmt, result};
 use arch::x86_64::interrupts;
 use arch::x86_64::msr::SetMSRsError;
 use arch::x86_64::regs::{SetupFpuError, SetupRegistersError, SetupSpecialRegistersError};
-use cpuid::{c3, filter_cpuid, t2, t2s, VmSpec};
+use cpuid::{c3, filter_cpuid, msrs_to_save_by_cpuid, t2, t2s, VmSpec};
 use kvm_bindings::{
     kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs,
     kvm_xsave, CpuId, MsrList, Msrs,
@@ -267,6 +267,7 @@ impl KvmVcpu {
             })
             .map_err(KvmVcpuConfigureError::FilterCpuid)?;
 
+        // Update the CPUID based on the template
         match vcpu_config.cpu_template {
             CpuFeaturesTemplate::T2 => t2::set_cpuid_entries(&mut cpuid, &cpuid_vm_spec)
                 .map_err(KvmVcpuConfigureError::SetCpuidEntries)?,
@@ -281,9 +282,33 @@ impl KvmVcpu {
             .set_cpuid2(&cpuid)
             .map_err(KvmVcpuConfigureError::SetCpuid)?;
 
-        // Set MSRs
+        // Initialize some architectural MSRs that will be set for boot.
         let mut msr_boot_entries = arch::x86_64::msr::create_boot_msr_entries();
 
+        // By this point the Guest CPUID is established. Some CPU features require MSRs
+        // to configure and interact with those features. If a MSR is writable from
+        // inside the Guest, or is changed by KVM or Firecracker on behalf of the Guest,
+        // then we will need to save it every time we take a snapshot, and restore its
+        // value when we restore the microVM since the Guest may need that value.
+        // Since CPUID tells us what features are enabled for the Guest, we can infer
+        // the extra MSRs that we need to save based on a dependency map.
+        let extra_msrs =
+            msrs_to_save_by_cpuid(&cpuid).map_err(KvmVcpuConfigureError::FilterCpuid)?;
+        for msr in extra_msrs {
+            self.msr_list
+                .push(msr)
+                .map_err(KvmVcpuConfigureError::PushMsrEntries)?;
+        }
+
+        // TODO: Some MSRs depend on values of other MSRs. This dependency will need to
+        // be implemented. For now we define known dependencies statically in the CPU
+        // templates.
+
+        // Depending on which CPU template the user selected, we may need to initialize
+        // additional MSRs for boot to correctly enable some CPU features. As stated in
+        // the previous comment, we get from the template a static list of MSRs we need
+        // to save at snapshot as well.
+        // C3 and T2 currently don't have extra MSRs to save/set
         if vcpu_config.cpu_template == CpuFeaturesTemplate::T2S {
             for msr in t2s::msr_entries_to_save() {
                 self.msr_list
@@ -292,6 +317,9 @@ impl KvmVcpu {
             }
             t2s::update_msr_entries(&mut msr_boot_entries);
         }
+        // By this point we know that at snapshot, the list of MSRs we need to
+        // save is `architectural MSRs` + `MSRs inferred through CPUID` + `other
+        // MSRs defined by the template`
 
         arch::x86_64::msr::set_msrs(&self.fd, &msr_boot_entries)?;
         arch::x86_64::regs::setup_regs(&self.fd, kernel_start_addr.raw_value() as u64)?;