Improve pci abstractions; support MMIO PCI access for aarch64 (#1019)

* Support PCI on aarch64 via MMIO access mechanisms.

* Redesign the PCI addressing code to be very clear and use more types.
  Currently some checks are done at runtime with assertions,
  but these could easily be moved to compile time by introducing
  traits about register sizes that can be implemented by each `RegisterSpan`
  within the `PciRegister` type.

* Note: PCI currently uses Port I/O on x86 and MMIO on aarch64 to read/write,
  but x86_64 may also use MMIO-based access in the future, because Port I/O
  is the legacy method to access the PCI configuration space.

* The `device_manager` now initializes and scans the PCI bus on
  both the aarch64 and x86_64 platforms. 
  * Scanning the PCI bus for the first time also maps the memory
    behind the configuration space.

Co-authored-by: Kevin Boos <[email protected]>

Author: NathanRoyer

Cargo.lock

@@ -933,6 +933,9 @@ else ifeq ($(ARCH),aarch64)
 	QEMU_FLAGS += -machine virt,gic-version=3
 	QEMU_FLAGS += -device ramfb
 	QEMU_FLAGS += -cpu cortex-a72
+	QEMU_FLAGS += -usb
+	QEMU_FLAGS += -device usb-ehci,id=ehci
+	QEMU_FLAGS += -device usb-kbd
 else
 	QEMU_FLAGS += -cpu Broadwell
 endif

Makefile

@@ -2,7 +2,7 @@
 
 use super::{
     InterruptControllerConfig::GicV3, GicV3InterruptControllerConfig,
-    BoardConfig, mpidr::DefinedMpidrValue,
+    BoardConfig, mpidr::DefinedMpidrValue, PciEcamConfig,
 };
 use memory_structs::PhysicalAddress;
 
@@ -38,4 +38,11 @@ pub const BOARD_CONFIG: BoardConfig = BoardConfig {
     pl011_base_addresses: [ PhysicalAddress::new_canonical(0x09000000) ],
     pl011_rx_spi: 33,
     cpu_local_timer_ppi: 30,
+
+    // obtained via internal qemu debugging
+    // todo: will this always be correct?
+    pci_ecam: PciEcamConfig {
+        base_address: PhysicalAddress::new_canonical(0x4010000000),
+        size_bytes: 0x10000000,
+    }
 };

kernel/arm_boards/src/boards/qemu_virt.rs

@@ -19,6 +19,12 @@ pub struct GicV3InterruptControllerConfig {
     pub redistributor_base_addresses: [PhysicalAddress; NUM_CPUS],
 }
 
+#[derive(Debug, Copy, Clone)]
+pub struct PciEcamConfig {
+    pub base_address: PhysicalAddress,
+    pub size_bytes: usize,
+}
+
 #[derive(Debug, Copy, Clone)]
 pub enum InterruptControllerConfig {
     GicV3(GicV3InterruptControllerConfig),
@@ -46,6 +52,8 @@ pub struct BoardConfig {
     //
     // aarch64 manuals define the default timer IRQ number to be 30.
     pub cpu_local_timer_ppi: u8,
+
+    pub pci_ecam: PciEcamConfig,
 }
 
 // by default & on x86_64, the default.rs file is used

kernel/arm_boards/src/lib.rs

@@ -12,6 +12,7 @@ event_types = { path = "../event_types" }
 serial_port = { path = "../serial_port" }
 console = { path = "../console" }
 logger = { path = "../logger" }
+pci = { path = "../pci" }
 derive_more = "0.99.0"
 mpmc = "0.1.6"
 log = "0.4.8"
@@ -20,7 +21,6 @@ log = "0.4.8"
 memory = { path = "../memory" }
 e1000 = { path = "../e1000" }
 acpi = { path = "../acpi" }
-pci = { path = "../pci" }
 ps2 = { path = "../ps2" }
 keyboard = { path = "../keyboard" }
 mouse = { path = "../mouse" }

kernel/device_manager/Cargo.toml

@@ -3,11 +3,11 @@
 
 extern crate alloc;
 
-use log::info;
+use log::{info, debug};
 
 #[cfg(target_arch = "x86_64")]
 use {
-    log::{error, debug, warn},
+    log::{error, warn},
     mpmc::Queue,
     event_types::Event,
     memory::MemoryManagementInfo,
@@ -86,19 +86,20 @@ pub fn init(
         mouse::init(ps2_controller.mouse_ref(), mouse_producer)?;
     }
 
-    // No PCI support on aarch64 at the moment
-    #[cfg(target_arch = "x86_64")] {
     // Initialize/scan the PCI bus to discover PCI devices
-    for dev in pci::pci_device_iter() {
-        debug!("Found pci device: {:X?}", dev);
+    for dev in pci::pci_device_iter()? {
+        debug!("Found PCI device: {:X?}", dev);
     }
 
+    // No NIC support on aarch64 at the moment
+    #[cfg(target_arch = "x86_64")] {
+
     // store all the initialized ixgbe NICs here to be added to the network interface list
     let mut ixgbe_devs = Vec::new();
 
     // Iterate over all PCI devices and initialize the drivers for the devices we support.
 
-    for dev in pci::pci_device_iter() {
+    for dev in pci::pci_device_iter()? {
         // Currently we skip Bridge devices, since we have no use for them yet. 
         if dev.class == 0x06 {
             continue;

kernel/device_manager/src/lib.rs

@@ -1,7 +1,7 @@
 [package]
 authors = ["Kevin Boos <[email protected]>"]
 name = "pci"
-description = "Basic PCI support for Theseus, x86 only."
+description = "Basic PCI support for Theseus."
 version = "0.1.0"
 edition = "2021"
 
@@ -12,10 +12,15 @@ log = "0.4.8"
 volatile = "0.2.4"
 zerocopy = "0.5.0"
 
-port_io = { path = "../../libs/port_io" }
 memory = { path = "../memory" }
 cpu = { path = "../cpu" }
 interrupts = { path = "../interrupts" }
 
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+port_io = { path = "../../libs/port_io" }
+
+[target.'cfg(target_arch = "aarch64")'.dependencies]
+arm_boards = { path = "../arm_boards" }
+
 [lib]
 crate-type = ["rlib"]