From 8b78f39c6fa39236d8c470795ce4a41e1a00eed3 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 6 Jan 2026 13:38:32 +0900 Subject: [PATCH 01/16] fix(interrupt remap): fix the initialization of interrupt remapping 1. do not use the global status register value for the global command register 2. wait untile enabled after updatating the global command register 3. print more registers for debugging Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 55 ++++++++++++++++--- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index bb4667ac8..179d51d34 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -1,6 +1,7 @@ //! Interrupt remapping of Vt-d. use acpi::AcpiTables; +use alloc::format; use crate::{ addr::{phy_addr::PhyAddr, virt_addr::VirtAddr, Addr}, @@ -60,6 +61,7 @@ mod registers { } } + mmio_r!(offset 0x10 => pub EXTENDED_CAPABILITY); mmio_w!(offset 0x18 => pub GLOBAL_COMMAND); mmio_r!(offset 0x1c => pub GLOBAL_STATUS); mmio_rw!(offset 0xb8 => pub IRTA); @@ -284,18 +286,53 @@ fn set_irta( registers::IRTA.write(val, vt_d_base.as_usize()); - let mut stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); - stat |= registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE; + let stat = registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE; registers::GLOBAL_COMMAND.write(stat, vt_d_base.as_usize()); + // Wait until enabled + for _ in 0..100 { + let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + if stat.contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE) + { + break; + } + } + + if !registers::GLOBAL_STATUS + .read(vt_d_base.as_usize()) + .contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE) + { + log::error!("Failed to enable Vt-d Interrupt Remapping."); + return; + } + + let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + + let mut scope_str = format!(""); + + for scope in drhd.device_scopes() { + let (device_scope, path) = scope; + scope_str = format!( + "{scope_str}\r\nVt-d Device Scope: entry type = {}, length = {}, flags = 0x{:x}, enumeration ID = {}, start bus number = {}, path = {path:x?}", + device_scope.entry_type, + device_scope.length, + device_scope.flags, + device_scope.enumeration_id, + device_scope.start_bus_number, + ); + } + log::info!( - "Vt-d Interrupt Remapping: Segment = {}, Vt-d Base = 0x{:x}, Table PhyAddr = 0x{:x}, enabled = {}, mode = {}", - segment_number, - drhd.register_base_address as usize, - phy_addr.as_usize(), - stat.contains(registers::GlobalCommandStatus::IRE | registers::GlobalCommandStatus::IRTP), - if is_x2apic { "x2APIC" } else { "xAPIC" }, - ) + "Vt-d Interrupt Remapping: Segment = {segment_number}, Vt-d Base = 0x{:x}, Table PhyAddr = 0x{:x}, enabled = {}, mode = {}, extended capability = 0x{:x}, global status = 0x{:x}, IRTA = 0x{:x}, DHRD.flags = 0x{:x}{scope_str}", + drhd.register_base_address as usize, + phy_addr.as_usize(), + stat.contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE), + if is_x2apic { "x2APIC" } else { "xAPIC" }, + registers::EXTENDED_CAPABILITY.read(vt_d_base.as_usize()), + stat.bits(), + registers::IRTA.read(vt_d_base.as_usize()), + drhd.flags + ) } pub fn allocate_remapping_entry( From b3907017c59ffcb8788db4af48938ba46211b603 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 6 Jan 2026 15:28:56 +0900 Subject: [PATCH 02/16] feat(iommu): initialize invalidate queue Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 123 +++++++++++++++--- 1 file changed, 106 insertions(+), 17 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 179d51d34..06cf8340b 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -2,6 +2,7 @@ use acpi::AcpiTables; use alloc::format; +use alloc::string::String; use crate::{ addr::{phy_addr::PhyAddr, virt_addr::VirtAddr, Addr}, @@ -18,6 +19,30 @@ use super::acpi::AcpiMapper; const TABLE_SIZE: usize = 256 * 4096; // 1MiB const TABLE_ENTRY_NUM: usize = TABLE_SIZE / 16; +/// Error type for Vt-d interrupt remapping initialization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VtdError { + /// Failed to enable Interrupt Remapping Table Pointer. + IrtpEnableFailed, + /// Failed to enable Interrupt Remapping. + IreEnableFailed, + /// Failed to enable Queued Invalidation. + QieEnableFailed, + /// Failed to allocate DMA memory. + DmaAllocationFailed, +} + +impl core::fmt::Display for VtdError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + VtdError::IrtpEnableFailed => write!(f, "Failed to enable IRTP"), + VtdError::IreEnableFailed => write!(f, "Failed to enable IRE"), + VtdError::QieEnableFailed => write!(f, "Failed to enable QIE"), + VtdError::DmaAllocationFailed => write!(f, "DMA allocation failed"), + } + } +} + #[allow(dead_code)] mod registers { use bitflags::bitflags; @@ -64,6 +89,9 @@ mod registers { mmio_r!(offset 0x10 => pub EXTENDED_CAPABILITY); mmio_w!(offset 0x18 => pub GLOBAL_COMMAND); mmio_r!(offset 0x1c => pub GLOBAL_STATUS); + mmio_rw!(offset 0x80 => pub IQH); // Invalidation Queue Head + mmio_rw!(offset 0x88 => pub IQT); // Invalidation Queue Tail + mmio_rw!(offset 0x90 => pub IQA); // Invalidation Queue Address mmio_rw!(offset 0xb8 => pub IRTA); } @@ -218,11 +246,11 @@ pub unsafe fn init_interrupt_remap( phy_offset: VirtAddr, acpi: &AcpiTables, is_x2apic: bool, -) -> Result<(), &'static str> { +) -> Result<(), VtdError> { let mut remap_table = [None; 32]; if let Ok(dmar) = acpi.find_table::() { - dmar.entries().for_each(|entry| { + for entry in dmar.entries() { if let DmarEntry::Drhd(drhd) = entry { drhd.device_scopes() .find(|(scope, _path)| scope.entry_type == 1); @@ -230,25 +258,25 @@ pub unsafe fn init_interrupt_remap( if let Some((phy_addr, _virt_addr)) = &remap_table[drhd.segment_number as usize] { let segment_number = drhd.segment_number as usize; // Set Interrupt Remapping Table Address Register - set_irta(segment_number, phy_offset, drhd, *phy_addr, is_x2apic); + set_irta(segment_number, phy_offset, drhd, *phy_addr, is_x2apic)?; } else { let segment_number = drhd.segment_number as usize; let pool = DMAPool::<[u8; TABLE_SIZE]>::new(segment_number, TABLE_SIZE / PAGESIZE) - .expect("DMAPool::new() failed."); + .ok_or(VtdError::DmaAllocationFailed)?; let virt_addr = pool.get_virt_addr(); let phy_addr = pool.get_phy_addr(); pool.leak(); // Set Interrupt Remapping Table Address Register - set_irta(segment_number, phy_offset, drhd, phy_addr, is_x2apic); + set_irta(segment_number, phy_offset, drhd, phy_addr, is_x2apic)?; remap_table[segment_number] = Some((phy_addr, virt_addr)); } } - }); + } } // Set INTERRUPT_REMAPPING @@ -277,7 +305,7 @@ fn set_irta( drhd: &DmarDrhd, phy_addr: PhyAddr, is_x2apic: bool, -) { +) -> Result<(), VtdError> { let vt_d_base = phy_offset + drhd.register_base_address as usize; // Extended Interrupt Mode (x2APIC) Enable @@ -298,17 +326,23 @@ fn set_irta( } } - if !registers::GLOBAL_STATUS - .read(vt_d_base.as_usize()) - .contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE) - { + let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + if !status.contains(registers::GlobalCommandStatus::IRTP) { + log::error!("Failed to enable Vt-d Interrupt Remapping Table Pointer."); + return Err(VtdError::IrtpEnableFailed); + } + if !status.contains(registers::GlobalCommandStatus::IRE) { log::error!("Failed to enable Vt-d Interrupt Remapping."); - return; + return Err(VtdError::IreEnableFailed); } - let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + // Initialize Invalidation Queue + init_invalidation_queue(segment_number, vt_d_base)?; + + // Re-read status after queue initialization + let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); - let mut scope_str = format!(""); + let mut scope_str = String::new(); for scope in drhd.device_scopes() { let (device_scope, path) = scope; @@ -326,13 +360,68 @@ fn set_irta( "Vt-d Interrupt Remapping: Segment = {segment_number}, Vt-d Base = 0x{:x}, Table PhyAddr = 0x{:x}, enabled = {}, mode = {}, extended capability = 0x{:x}, global status = 0x{:x}, IRTA = 0x{:x}, DHRD.flags = 0x{:x}{scope_str}", drhd.register_base_address as usize, phy_addr.as_usize(), - stat.contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE), + status.contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE), if is_x2apic { "x2APIC" } else { "xAPIC" }, registers::EXTENDED_CAPABILITY.read(vt_d_base.as_usize()), - stat.bits(), + status.bits(), registers::IRTA.read(vt_d_base.as_usize()), drhd.flags - ) + ); + + Ok(()) +} + +fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result<(), VtdError> { + // Reset Invalidation Queue Tail + registers::IQT.write(0, vt_d_base.as_usize()); + + // Allocate a page for the Invalidation Queue + let pool = + DMAPool::<[u8; PAGESIZE]>::new(segment_number, 1).ok_or(VtdError::DmaAllocationFailed)?; + + let phy_addr = pool.get_phy_addr().as_usize() as u64; + + // Set Invalidation Queue Address Register + // Bits [11:0] = Queue Size (0 = 4KB, 1 page) + // Bits [63:12] = Base Address + // Queue size = 0 (4KB, 256 entries) + let iqa_value = phy_addr & !((1 << 12) - 1); + registers::IQA.write(iqa_value, vt_d_base.as_usize()); + + // Leak the pool to keep it allocated + pool.leak(); + + // Enable Queued Invalidation + let stat = registers::GlobalCommandStatus::QIE; + registers::GLOBAL_COMMAND.write(stat, vt_d_base.as_usize()); + + // Wait until Queued Invalidation is enabled + for _ in 0..100 { + let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + if stat.contains(registers::GlobalCommandStatus::QIE) { + break; + } + } + + if !registers::GLOBAL_STATUS + .read(vt_d_base.as_usize()) + .contains(registers::GlobalCommandStatus::QIE) + { + log::error!( + "Failed to enable Vt-d Queued Invalidation for segment {}.", + segment_number + ); + return Err(VtdError::QieEnableFailed); + } + + log::info!( + "Vt-d Queued Invalidation enabled: Segment = {}, Queue PhyAddr = 0x{:x}, IQA = 0x{:x}", + segment_number, + phy_addr, + registers::IQA.read(vt_d_base.as_usize()) + ); + + Ok(()) } pub fn allocate_remapping_entry( From 6e350be0ea309a2c9b5e0ad37f72eb243af193eb Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 6 Jan 2026 15:39:02 +0900 Subject: [PATCH 03/16] fix(iommu): print IQH and IQT of QIE Signed-off-by: Yuuki Takano --- awkernel_lib/src/arch/x86_64/interrupt_remap.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 06cf8340b..b3f647909 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -415,10 +415,12 @@ fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result } log::info!( - "Vt-d Queued Invalidation enabled: Segment = {}, Queue PhyAddr = 0x{:x}, IQA = 0x{:x}", + "Vt-d Queued Invalidation enabled: Segment = {}, Queue PhyAddr = 0x{:x}, IQA = 0x{:x}, IQH = {}, IQT = {}", segment_number, phy_addr, - registers::IQA.read(vt_d_base.as_usize()) + registers::IQA.read(vt_d_base.as_usize()), + registers::IQH.read(vt_d_base.as_usize()), + registers::IQT.read(vt_d_base.as_usize()) ); Ok(()) From 5eca6d2d4425d0734765b78055d458ff2ad2ff81 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 6 Jan 2026 16:41:06 +0900 Subject: [PATCH 04/16] fix(iommu): add update_global_command() Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 146 ++++++++++++------ 1 file changed, 100 insertions(+), 46 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index b3f647909..392472485 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -30,6 +30,8 @@ pub enum VtdError { QieEnableFailed, /// Failed to allocate DMA memory. DmaAllocationFailed, + /// Timeout waiting for command completion. + Timeout, } impl core::fmt::Display for VtdError { @@ -39,6 +41,7 @@ impl core::fmt::Display for VtdError { VtdError::IreEnableFailed => write!(f, "Failed to enable IRE"), VtdError::QieEnableFailed => write!(f, "Failed to enable QIE"), VtdError::DmaAllocationFailed => write!(f, "DMA allocation failed"), + VtdError::Timeout => write!(f, "Timeout waiting for command completion"), } } } @@ -298,6 +301,93 @@ pub unsafe fn init_interrupt_remap( Ok(()) } +/// Update GLOBAL_COMMAND register following Intel VT-d specification. +/// +/// Steps: +/// 1. Read GLOBAL_STATUS register +/// 2. Reset one-shot bits (keeping only persistent bits) +/// 3. Set/clear the target bit +/// 4. Write to GLOBAL_COMMAND register +/// 5. Wait until GLOBAL_STATUS indicates command is serviced +fn update_global_command( + vt_d_base: VirtAddr, + bit_to_set: registers::GlobalCommandStatus, + enable: bool, +) -> Result<(), VtdError> { + // Step 1: Read GLOBAL_STATUS + let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + + // Step 2: Reset one-shot bits (preserve only bits 31, 30, 27, 26, 25, 24, 23) + // One-shot bits mask: 0x96FFFFFF (bits that should be preserved) + let persistent_mask = registers::GlobalCommandStatus::from_bits_truncate(0x96FF_FFFF); + let status_persistent = status & persistent_mask; + + // Step 3: Set or clear the target bit + let command = if enable { + status_persistent | bit_to_set + } else { + status_persistent & !bit_to_set + }; + + // Step 4: Write to GLOBAL_COMMAND + registers::GLOBAL_COMMAND.write(command, vt_d_base.as_usize()); + + // Step 5: Wait until GLOBAL_STATUS indicates command is serviced + if wait_toggle_then_set(vt_d_base, bit_to_set, enable).is_err() { + // Timeout: bit not updated as expected + let current = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); + Err( + if bit_to_set.contains(registers::GlobalCommandStatus::IRTP) + && !current.contains(registers::GlobalCommandStatus::IRTP) + { + VtdError::IrtpEnableFailed + } else if bit_to_set.contains(registers::GlobalCommandStatus::IRE) + && !current.contains(registers::GlobalCommandStatus::IRE) + { + VtdError::IreEnableFailed + } else if bit_to_set.contains(registers::GlobalCommandStatus::QIE) + && !current.contains(registers::GlobalCommandStatus::QIE) + { + VtdError::QieEnableFailed + } else { + VtdError::Timeout // Default fallback + }, + ) + } else { + Ok(()) + } +} + +fn wait_toggle_then_set( + vt_d_base: VirtAddr, + status_bit: registers::GlobalCommandStatus, + enable: bool, +) -> Result<(), VtdError> { + if enable { + // wait until 1 + for _ in 0..1000 { + if registers::GLOBAL_STATUS + .read(vt_d_base.as_usize()) + .contains(status_bit) + { + return Ok(()); + } + } + } else { + // wait until 0 + for _ in 0..1000 { + if !registers::GLOBAL_STATUS + .read(vt_d_base.as_usize()) + .contains(status_bit) + { + return Ok(()); + } + } + } + + Err(VtdError::Timeout) +} + /// Set Interrupt Remapping Table Address Register. fn set_irta( segment_number: usize, @@ -314,27 +404,11 @@ fn set_irta( registers::IRTA.write(val, vt_d_base.as_usize()); - let stat = registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE; - registers::GLOBAL_COMMAND.write(stat, vt_d_base.as_usize()); + // Enable IRTP (Interrupt Remapping Table Pointer) + update_global_command(vt_d_base, registers::GlobalCommandStatus::IRTP, true)?; - // Wait until enabled - for _ in 0..100 { - let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); - if stat.contains(registers::GlobalCommandStatus::IRTP | registers::GlobalCommandStatus::IRE) - { - break; - } - } - - let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); - if !status.contains(registers::GlobalCommandStatus::IRTP) { - log::error!("Failed to enable Vt-d Interrupt Remapping Table Pointer."); - return Err(VtdError::IrtpEnableFailed); - } - if !status.contains(registers::GlobalCommandStatus::IRE) { - log::error!("Failed to enable Vt-d Interrupt Remapping."); - return Err(VtdError::IreEnableFailed); - } + // Enable IRE (Interrupt Remapping Enable) + update_global_command(vt_d_base, registers::GlobalCommandStatus::IRE, true)?; // Initialize Invalidation Queue init_invalidation_queue(segment_number, vt_d_base)?; @@ -381,38 +455,18 @@ fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result let phy_addr = pool.get_phy_addr().as_usize() as u64; - // Set Invalidation Queue Address Register - // Bits [11:0] = Queue Size (0 = 4KB, 1 page) - // Bits [63:12] = Base Address - // Queue size = 0 (4KB, 256 entries) + // bits[63:12] = Base (4KB aligned) + // bit[11] = DW (128/256) + // bits[2:0] = queue size (0 = 4KB) + // bits[10:3] = Reserved let iqa_value = phy_addr & !((1 << 12) - 1); registers::IQA.write(iqa_value, vt_d_base.as_usize()); // Leak the pool to keep it allocated pool.leak(); - // Enable Queued Invalidation - let stat = registers::GlobalCommandStatus::QIE; - registers::GLOBAL_COMMAND.write(stat, vt_d_base.as_usize()); - - // Wait until Queued Invalidation is enabled - for _ in 0..100 { - let stat = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); - if stat.contains(registers::GlobalCommandStatus::QIE) { - break; - } - } - - if !registers::GLOBAL_STATUS - .read(vt_d_base.as_usize()) - .contains(registers::GlobalCommandStatus::QIE) - { - log::error!( - "Failed to enable Vt-d Queued Invalidation for segment {}.", - segment_number - ); - return Err(VtdError::QieEnableFailed); - } + // Enable Queued Invalidation (QIE) + update_global_command(vt_d_base, registers::GlobalCommandStatus::QIE, true)?; log::info!( "Vt-d Queued Invalidation enabled: Segment = {}, Queue PhyAddr = 0x{:x}, IQA = 0x{:x}, IQH = {}, IQT = {}", From 5b2be9499d5641c1a7301e98081c3af8e76738b5 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 6 Jan 2026 16:51:31 +0900 Subject: [PATCH 05/16] fix(iommu): Wait for register-based invalidation to complete Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 392472485..05edea17c 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -95,6 +95,7 @@ mod registers { mmio_rw!(offset 0x80 => pub IQH); // Invalidation Queue Head mmio_rw!(offset 0x88 => pub IQT); // Invalidation Queue Tail mmio_rw!(offset 0x90 => pub IQA); // Invalidation Queue Address + mmio_r!(offset 0x9c => pub ICS); // Invalidation Completion Status mmio_rw!(offset 0xb8 => pub IRTA); } @@ -388,6 +389,24 @@ fn wait_toggle_then_set( Err(VtdError::Timeout) } +/// Wait for register-based invalidation to complete. +/// This must be called before enabling Queued Invalidation. +fn wait_register_based_invalidation_complete(vt_d_base: VirtAddr) -> Result<(), VtdError> { + const ICS_IWC: u32 = 1; // Invalidation Wait Descriptor Complete + + // Wait for any pending register-based invalidation to complete + // by checking the ICS register's IWC bit is 0 + for _ in 0..1000 { + let ics = registers::ICS.read(vt_d_base.as_usize()); + if (ics & ICS_IWC) == 0 { + return Ok(()); + } + } + + log::error!("Timeout waiting for register-based invalidation to complete"); + Err(VtdError::Timeout) +} + /// Set Interrupt Remapping Table Address Register. fn set_irta( segment_number: usize, @@ -446,6 +465,10 @@ fn set_irta( } fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result<(), VtdError> { + // Wait for any pending register-based invalidation to complete + // before enabling Queued Invalidation + wait_register_based_invalidation_complete(vt_d_base)?; + // Reset Invalidation Queue Tail registers::IQT.write(0, vt_d_base.as_usize()); From 1bcc26ffb00d82d213c5708d63f3421b284cd7e8 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Wed, 7 Jan 2026 17:05:30 +0900 Subject: [PATCH 06/16] fix(vt-d): introduce `IommuInfo` structure to maintain Vt-d Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 05edea17c..f81b927ea 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -1,8 +1,7 @@ //! Interrupt remapping of Vt-d. use acpi::AcpiTables; -use alloc::format; -use alloc::string::String; +use alloc::{collections::linked_list::LinkedList, format, string::String}; use crate::{ addr::{phy_addr::PhyAddr, virt_addr::VirtAddr, Addr}, @@ -153,8 +152,25 @@ impl IRTEntry { } } -static INTERRUPT_REMAPPING: [Mutex>; 32] = - array![_ => Mutex::new(None); 32]; +static IOMMU: [IommuInfo; 32] = array![x => + IommuInfo { + segment_number: x, + vtd_units: Mutex::new(LinkedList::new()), + interrupt_remapping: Mutex::new(None), + }; 32]; + +#[allow(dead_code)] // TODO: remove this +struct IommuInfo { + segment_number: usize, + vtd_units: Mutex>, + interrupt_remapping: Mutex>, +} + +#[allow(dead_code)] // TODO: remove this +struct VtdAddrs { + vt_d_base: VirtAddr, + iq_base: VirtAddr, // Cache Invalidation Queue base address +} #[derive(Debug)] pub struct RemapInfo { @@ -171,7 +187,9 @@ impl RemapInfo { impl Drop for RemapInfo { fn drop(&mut self) { let mut node = MCSNode::new(); - let mut table = INTERRUPT_REMAPPING[self.segment_number].lock(&mut node); + let mut table = IOMMU[self.segment_number] + .interrupt_remapping + .lock(&mut node); if let Some(table) = table.as_mut() { table.deallocate_entry(self.entry_id); @@ -262,7 +280,8 @@ pub unsafe fn init_interrupt_remap( if let Some((phy_addr, _virt_addr)) = &remap_table[drhd.segment_number as usize] { let segment_number = drhd.segment_number as usize; // Set Interrupt Remapping Table Address Register - set_irta(segment_number, phy_offset, drhd, *phy_addr, is_x2apic)?; + let vt_d_base = phy_offset + drhd.register_base_address as usize; + set_irta(segment_number, vt_d_base, drhd, *phy_addr, is_x2apic)?; } else { let segment_number = drhd.segment_number as usize; @@ -275,7 +294,8 @@ pub unsafe fn init_interrupt_remap( pool.leak(); // Set Interrupt Remapping Table Address Register - set_irta(segment_number, phy_offset, drhd, phy_addr, is_x2apic)?; + let vt_d_base = phy_offset + drhd.register_base_address as usize; + set_irta(segment_number, vt_d_base, drhd, phy_addr, is_x2apic)?; remap_table[segment_number] = Some((phy_addr, virt_addr)); } @@ -283,7 +303,7 @@ pub unsafe fn init_interrupt_remap( } } - // Set INTERRUPT_REMAPPING + // Set interrupt_remapping in IOMMU for (i, entry) in remap_table.iter().enumerate() { if let Some((_phy_addr, virt_addr)) = entry { let mut table = InterruptRemapping { @@ -295,7 +315,7 @@ pub unsafe fn init_interrupt_remap( table.disable_all(); let mut node = MCSNode::new(); - INTERRUPT_REMAPPING[i].lock(&mut node).replace(table); + IOMMU[i].interrupt_remapping.lock(&mut node).replace(table); } } @@ -410,13 +430,11 @@ fn wait_register_based_invalidation_complete(vt_d_base: VirtAddr) -> Result<(), /// Set Interrupt Remapping Table Address Register. fn set_irta( segment_number: usize, - phy_offset: VirtAddr, + vt_d_base: VirtAddr, drhd: &DmarDrhd, phy_addr: PhyAddr, is_x2apic: bool, ) -> Result<(), VtdError> { - let vt_d_base = phy_offset + drhd.register_base_address as usize; - // Extended Interrupt Mode (x2APIC) Enable let eime = if is_x2apic { 1 << 11 } else { 0 }; let val = (phy_addr.as_usize() as u64 & !((1 << 12) - 1)) | eime | 0b1111; // Size @@ -430,7 +448,16 @@ fn set_irta( update_global_command(vt_d_base, registers::GlobalCommandStatus::IRE, true)?; // Initialize Invalidation Queue - init_invalidation_queue(segment_number, vt_d_base)?; + let iq_base = init_invalidation_queue(segment_number, vt_d_base)?; + + // Store VT-d unit info + let vtd_addrs = VtdAddrs { vt_d_base, iq_base }; + + let mut node = MCSNode::new(); + IOMMU[segment_number] + .vtd_units + .lock(&mut node) + .push_back(vtd_addrs); // Re-read status after queue initialization let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); @@ -464,7 +491,10 @@ fn set_irta( Ok(()) } -fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result<(), VtdError> { +fn init_invalidation_queue( + segment_number: usize, + vt_d_base: VirtAddr, +) -> Result { // Wait for any pending register-based invalidation to complete // before enabling Queued Invalidation wait_register_based_invalidation_complete(vt_d_base)?; @@ -477,6 +507,7 @@ fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result DMAPool::<[u8; PAGESIZE]>::new(segment_number, 1).ok_or(VtdError::DmaAllocationFailed)?; let phy_addr = pool.get_phy_addr().as_usize() as u64; + let virt_addr = pool.get_virt_addr(); // bits[63:12] = Base (4KB aligned) // bit[11] = DW (128/256) @@ -500,7 +531,7 @@ fn init_invalidation_queue(segment_number: usize, vt_d_base: VirtAddr) -> Result registers::IQT.read(vt_d_base.as_usize()) ); - Ok(()) + Ok(virt_addr) } pub fn allocate_remapping_entry( @@ -511,7 +542,7 @@ pub fn allocate_remapping_entry( lowest_priority: bool, ) -> Option { let mut node = MCSNode::new(); - let mut table = INTERRUPT_REMAPPING[segment_number].lock(&mut node); + let mut table = IOMMU[segment_number].interrupt_remapping.lock(&mut node); if let Some(table) = table.as_mut() { table.allocate_entry(dest_apic_id, irq, level_trigger, lowest_priority) From 2b7a27d1581fca1eabb5aeb0fcf734c278bbc651 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 15:45:52 +0900 Subject: [PATCH 07/16] feat(vt-d): implement cahce invalidation Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 221 +++++++++++++++++- 1 file changed, 220 insertions(+), 1 deletion(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index f81b927ea..7c41c4be2 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -31,6 +31,12 @@ pub enum VtdError { DmaAllocationFailed, /// Timeout waiting for command completion. Timeout, + /// Invalidation Queue is full. + QueueFull, + /// Invalidation operation failed. + InvalidationFailed, + /// Fault status detected (IQE/ITE/ICE). + FaultStatus(u32), } impl core::fmt::Display for VtdError { @@ -41,6 +47,9 @@ impl core::fmt::Display for VtdError { VtdError::QieEnableFailed => write!(f, "Failed to enable QIE"), VtdError::DmaAllocationFailed => write!(f, "DMA allocation failed"), VtdError::Timeout => write!(f, "Timeout waiting for command completion"), + VtdError::QueueFull => write!(f, "Invalidation Queue is full"), + VtdError::InvalidationFailed => write!(f, "Invalidation operation failed"), + VtdError::FaultStatus(status) => write!(f, "Fault status detected: 0x{:x}", status), } } } @@ -88,13 +97,23 @@ mod registers { } } + bitflags! { + #[derive(Debug, Clone, Copy)] + pub struct FaultStatus: u32 { + const IQE = 1 << 4; // Invalidation Queue Error + const ICE = 1 << 5; // Invalidation Completion Error + const ITE = 1 << 6; // Invalidation Time-out Error + } + } + mmio_r!(offset 0x10 => pub EXTENDED_CAPABILITY); mmio_w!(offset 0x18 => pub GLOBAL_COMMAND); mmio_r!(offset 0x1c => pub GLOBAL_STATUS); + mmio_rw!(offset 0x34 => pub FSTS); // Fault Status Register mmio_rw!(offset 0x80 => pub IQH); // Invalidation Queue Head mmio_rw!(offset 0x88 => pub IQT); // Invalidation Queue Tail mmio_rw!(offset 0x90 => pub IQA); // Invalidation Queue Address - mmio_r!(offset 0x9c => pub ICS); // Invalidation Completion Status + mmio_rw!(offset 0x9c => pub ICS); // Invalidation Completion Status (W1C for IWC) mmio_rw!(offset 0xb8 => pub IRTA); } @@ -534,6 +553,206 @@ fn init_invalidation_queue( Ok(virt_addr) } +/// Invalidation Queue Descriptor Types +#[repr(C, align(16))] +#[derive(Debug, Clone, Copy)] +struct InvalidationDescriptor { + lo: u64, + hi: u64, +} + +impl InvalidationDescriptor { + /// Create Interrupt Entry Cache Invalidation Descriptor + fn interrupt_entry_cache_invalidation(index: u16, index_mask: u16, granularity: u8) -> Self { + let lo = 0x04 // Type: Interrupt Entry Cache Invalidation + | ((granularity as u64) << 4) // Bits [4]: Granularity + | ((index_mask as u64) << 27) // Bits [31:27]: Interrupt Index Mask (IM) + | ((index as u64) << 32); // Bits [47:32]: Interrupt Index (IIDX) + Self { lo, hi: 0 } + } + + /// Create Invalidation Wait Descriptor + fn invalidation_wait(status_write: bool, status_data: u32, status_addr: u64) -> Self { + let lo = 0x05 // Type: Invalidation Wait Descriptor + | if status_write { 1 << 5 } else { 0 } // SW: Status Write + | (1 << 6) // FN: When Set, indicates descriptors following the invalidation wait descriptor must be processed by hardware only after the invalidation wait descriptor completes. + | ((status_data as u64) << 32); + let hi = status_addr; + Self { lo, hi } + } +} + +const QUEUE_SIZE_BYTES: usize = PAGESIZE; // 4KB +const DESCRIPTOR_SIZE: usize = 16; // 128-bit descriptor +const QUEUE_ENTRIES: usize = QUEUE_SIZE_BYTES / DESCRIPTOR_SIZE; // 256 entries + +/// Push a descriptor to the Invalidation Queue +fn push_invalidation_descriptor( + segment_number: usize, + vt_d_base: VirtAddr, + iq_base: VirtAddr, + descriptor: InvalidationDescriptor, +) -> Result<(), VtdError> { + // Read current head and tail + let head = registers::IQH.read(vt_d_base.as_usize()) as usize; + let tail = registers::IQT.read(vt_d_base.as_usize()) as usize; + + // Convert to entry indices (tail/head are byte offsets, 16B aligned) + let head_idx = head / DESCRIPTOR_SIZE; + let tail_idx = tail / DESCRIPTOR_SIZE; + + // Check if queue is full (next tail would catch up to head) + let next_tail_idx = (tail_idx + 1) % QUEUE_ENTRIES; + if next_tail_idx == head_idx { + log::error!( + "Invalidation Queue full: segment={}, head={}, tail={}", + segment_number, + head_idx, + tail_idx + ); + return Err(VtdError::QueueFull); + } + + // Write descriptor to IQ[tail] + let queue = unsafe { &mut *(iq_base.as_mut_ptr::<[InvalidationDescriptor; QUEUE_ENTRIES]>()) }; + queue[tail_idx] = descriptor; + + // Memory fence to ensure descriptor is visible before updating IQT + // This is critical for DMA coherency + core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + + // Update IQT (doorbell) - this is a byte offset, must be 16B aligned + let new_tail = (next_tail_idx * DESCRIPTOR_SIZE) as u64; + registers::IQT.write(new_tail, vt_d_base.as_usize()); + + log::trace!( + "Pushed descriptor to IQ: segment={}, tail_idx={} -> {}, descriptor={:?}", + segment_number, + tail_idx, + next_tail_idx, + descriptor + ); + + Ok(()) +} + +/// Wait for invalidation completion by checking ICS or using wait descriptor +fn wait_invalidation_complete(segment_number: usize, vt_d_base: VirtAddr) -> Result<(), VtdError> { + const ICS_IWC: u32 = 1; // Invalidation Wait Descriptor Complete + + // Poll ICS register for completion + for _ in 0..10000 { + let ics = registers::ICS.read(vt_d_base.as_usize()); + if (ics & ICS_IWC) != 0 { + // Clear IWC bit by writing 1 (W1C) + registers::ICS.write(ICS_IWC, vt_d_base.as_usize()); + return Ok(()); + } + + // Check for fault status + let fsts = registers::FSTS.read(vt_d_base.as_usize()); + let fault_flags = registers::FaultStatus::from_bits_truncate(fsts); + if fault_flags.intersects( + registers::FaultStatus::IQE | registers::FaultStatus::ITE | registers::FaultStatus::ICE, + ) { + log::error!( + "Invalidation fault detected: segment={}, FSTS=0x{:x}, fault_flags={:?}", + segment_number, + fsts, + fault_flags + ); + + // Clear fault status bits (W1C) + registers::FSTS.write(fsts, vt_d_base.as_usize()); + + return Err(VtdError::FaultStatus(fsts)); + } + } + + log::warn!( + "Invalidation wait timeout: segment={}, ICS=0x{:x}", + segment_number, + registers::ICS.read(vt_d_base.as_usize()) + ); + + Err(VtdError::Timeout) +} + +/// Invalidate interrupt entry cache for a specific index +pub fn invalidate_interrupt_entry( + segment_number: usize, + interrupt_index: u16, +) -> Result<(), VtdError> { + let mut node = MCSNode::new(); + let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); + + if let Some(vtd_addrs) = vtd_units.front() { + let vt_d_base = vtd_addrs.vt_d_base; + let iq_base = vtd_addrs.iq_base; + + // Create Interrupt Entry Cache Invalidation descriptor + // Granularity = 0 (global), index_mask = 0 (specific index) + let desc = + InvalidationDescriptor::interrupt_entry_cache_invalidation(interrupt_index, 0, 0); + + // Push descriptor to queue + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; + + // Create and push Invalidation Wait descriptor + let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; + + // Wait for completion + wait_invalidation_complete(segment_number, vt_d_base)?; + + log::debug!( + "Invalidated interrupt entry: segment={}, index={}", + segment_number, + interrupt_index + ); + + Ok(()) + } else { + log::error!("No VT-d unit found for segment {}", segment_number); + Err(VtdError::InvalidationFailed) + } +} + +/// Invalidate all interrupt entries (global invalidation) +pub fn invalidate_all_interrupt_entries(segment_number: usize) -> Result<(), VtdError> { + let mut node = MCSNode::new(); + let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); + + if let Some(vtd_addrs) = vtd_units.front() { + let vt_d_base = vtd_addrs.vt_d_base; + let iq_base = vtd_addrs.iq_base; + + // Create global Interrupt Entry Cache Invalidation descriptor + // Granularity = 1 (global) + let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(0, 0, 1); + + // Push descriptor to queue + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; + + // Create and push Invalidation Wait descriptor + let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; + + // Wait for completion + wait_invalidation_complete(segment_number, vt_d_base)?; + + log::info!( + "Invalidated all interrupt entries: segment={}", + segment_number + ); + + Ok(()) + } else { + log::error!("No VT-d unit found for segment {}", segment_number); + Err(VtdError::InvalidationFailed) + } +} + pub fn allocate_remapping_entry( segment_number: usize, dest_apic_id: u32, From 236036775cc4e90351a65d14725d165347df455b Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 16:00:49 +0900 Subject: [PATCH 08/16] fix(vt-d): update invalidate functions Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 95 ++++++++----------- 1 file changed, 41 insertions(+), 54 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 7c41c4be2..21bef3293 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -679,78 +679,65 @@ fn wait_invalidation_complete(segment_number: usize, vt_d_base: VirtAddr) -> Res } /// Invalidate interrupt entry cache for a specific index -pub fn invalidate_interrupt_entry( +fn invalidate_interrupt_entry( segment_number: usize, + vtd_addrs: &VtdAddrs, interrupt_index: u16, ) -> Result<(), VtdError> { - let mut node = MCSNode::new(); - let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); - - if let Some(vtd_addrs) = vtd_units.front() { - let vt_d_base = vtd_addrs.vt_d_base; - let iq_base = vtd_addrs.iq_base; + let vt_d_base = vtd_addrs.vt_d_base; + let iq_base = vtd_addrs.iq_base; - // Create Interrupt Entry Cache Invalidation descriptor - // Granularity = 0 (global), index_mask = 0 (specific index) - let desc = - InvalidationDescriptor::interrupt_entry_cache_invalidation(interrupt_index, 0, 0); + // Create Interrupt Entry Cache Invalidation descriptor + // Granularity = 0 (global), index_mask = 0 (specific index) + let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(interrupt_index, 0, 0); - // Push descriptor to queue - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; + // Push descriptor to queue + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; - // Create and push Invalidation Wait descriptor - let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; + // Create and push Invalidation Wait descriptor + let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; - // Wait for completion - wait_invalidation_complete(segment_number, vt_d_base)?; + // Wait for completion + wait_invalidation_complete(segment_number, vt_d_base)?; - log::debug!( - "Invalidated interrupt entry: segment={}, index={}", - segment_number, - interrupt_index - ); + log::debug!( + "Invalidated interrupt entry: segment={}, index={}", + segment_number, + interrupt_index + ); - Ok(()) - } else { - log::error!("No VT-d unit found for segment {}", segment_number); - Err(VtdError::InvalidationFailed) - } + Ok(()) } /// Invalidate all interrupt entries (global invalidation) -pub fn invalidate_all_interrupt_entries(segment_number: usize) -> Result<(), VtdError> { - let mut node = MCSNode::new(); - let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); - - if let Some(vtd_addrs) = vtd_units.front() { - let vt_d_base = vtd_addrs.vt_d_base; - let iq_base = vtd_addrs.iq_base; +fn invalidate_all_interrupt_entries( + segment_number: usize, + vtd_addrs: &VtdAddrs, +) -> Result<(), VtdError> { + let vt_d_base = vtd_addrs.vt_d_base; + let iq_base = vtd_addrs.iq_base; - // Create global Interrupt Entry Cache Invalidation descriptor - // Granularity = 1 (global) - let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(0, 0, 1); + // Create global Interrupt Entry Cache Invalidation descriptor + // Granularity = 1 (global) + let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(0, 0, 1); - // Push descriptor to queue - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; + // Push descriptor to queue + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; - // Create and push Invalidation Wait descriptor - let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; + // Create and push Invalidation Wait descriptor + let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; - // Wait for completion - wait_invalidation_complete(segment_number, vt_d_base)?; + // Wait for completion + wait_invalidation_complete(segment_number, vt_d_base)?; - log::info!( - "Invalidated all interrupt entries: segment={}", - segment_number - ); + log::info!( + "Invalidated all interrupt entries: segment={}", + segment_number + ); - Ok(()) - } else { - log::error!("No VT-d unit found for segment {}", segment_number); - Err(VtdError::InvalidationFailed) - } + Ok(()) } pub fn allocate_remapping_entry( From b6a5b5b4ec228f1ced2e3635fd9f2614a1aaf34b Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 17:36:23 +0900 Subject: [PATCH 09/16] fix(vt-d): invalidete cache when initializing Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 102 +++++++++++------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 21bef3293..9e9d751aa 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -106,6 +106,9 @@ mod registers { } } + // Extended Capability Register bits + pub const ECAP_ESIRTPS: u64 = 1 << 62; // Enhanced Set Interrupt Root Table Pointer Support + mmio_r!(offset 0x10 => pub EXTENDED_CAPABILITY); mmio_w!(offset 0x18 => pub GLOBAL_COMMAND); mmio_r!(offset 0x1c => pub GLOBAL_STATUS); @@ -338,6 +341,37 @@ pub unsafe fn init_interrupt_remap( } } + // Perform global invalidation for all VT-d units + // (unless ESIRTPS is supported, which handles this automatically) + for (segment_number, iommu_info) in IOMMU.iter().enumerate() { + let mut node = MCSNode::new(); + let vtd_units = iommu_info.vtd_units.lock(&mut node); + + if let Some(vtd_addrs) = vtd_units.front() { + let vt_d_base = vtd_addrs.vt_d_base; + let ecap = registers::EXTENDED_CAPABILITY.read(vt_d_base.as_usize()); + + // Check if ESIRTPS is supported (bit 2) + if (ecap & registers::ECAP_ESIRTPS) == 0 { + // ESIRTPS not supported, need to perform global invalidation + log::info!( + "VT-d segment {}: ESIRTPS not supported (ECAP=0x{:x}), performing global invalidation", + segment_number, + ecap + ); + + // Perform global invalidation using the vtd_addrs reference + invalidate_all_interrupt_entries(segment_number, vtd_addrs)?; + } else { + log::debug!( + "VT-d segment {}: ESIRTPS supported (ECAP=0x{:x}), skipping global invalidation", + segment_number, + ecap + ); + } + } + } + Ok(()) } @@ -442,7 +476,7 @@ fn wait_register_based_invalidation_complete(vt_d_base: VirtAddr) -> Result<(), } } - log::error!("Timeout waiting for register-based invalidation to complete"); + log::error!("Vt-d: Timeout waiting for register-based invalidation to complete"); Err(VtdError::Timeout) } @@ -605,7 +639,7 @@ fn push_invalidation_descriptor( let next_tail_idx = (tail_idx + 1) % QUEUE_ENTRIES; if next_tail_idx == head_idx { log::error!( - "Invalidation Queue full: segment={}, head={}, tail={}", + "Vt-d: Invalidation Queue full: segment = {}, head = {}, tail = {}", segment_number, head_idx, tail_idx @@ -626,7 +660,7 @@ fn push_invalidation_descriptor( registers::IQT.write(new_tail, vt_d_base.as_usize()); log::trace!( - "Pushed descriptor to IQ: segment={}, tail_idx={} -> {}, descriptor={:?}", + "Vt-d: Pushed descriptor to IQ: segment = {}, tail_idx = {} -> {}, descriptor = {:?}", segment_number, tail_idx, next_tail_idx, @@ -637,48 +671,42 @@ fn push_invalidation_descriptor( } /// Wait for invalidation completion by checking ICS or using wait descriptor -fn wait_invalidation_complete(segment_number: usize, vt_d_base: VirtAddr) -> Result<(), VtdError> { - const ICS_IWC: u32 = 1; // Invalidation Wait Descriptor Complete +fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdAddrs) -> Result<(), VtdError> { + let vt_d_base = vtd_addrs.vt_d_base; + let iq_base = vtd_addrs.iq_base; + + // Create and push Invalidation Wait descriptor + let mut dma_pool = DMAPool::<[u32; PAGESIZE / 4]>::new(segment_number, 1) + .ok_or(VtdError::DmaAllocationFailed)?; + + let status_addr = dma_pool.get_phy_addr().as_usize() as u64; + let status = dma_pool.as_mut(); + + status[0] = 0; + core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + + let wait_desc = InvalidationDescriptor::invalidation_wait(true, 1, status_addr); + push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; - // Poll ICS register for completion for _ in 0..10000 { - let ics = registers::ICS.read(vt_d_base.as_usize()); - if (ics & ICS_IWC) != 0 { - // Clear IWC bit by writing 1 (W1C) - registers::ICS.write(ICS_IWC, vt_d_base.as_usize()); + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + if status[0] != 0 { return Ok(()); } - - // Check for fault status - let fsts = registers::FSTS.read(vt_d_base.as_usize()); - let fault_flags = registers::FaultStatus::from_bits_truncate(fsts); - if fault_flags.intersects( - registers::FaultStatus::IQE | registers::FaultStatus::ITE | registers::FaultStatus::ICE, - ) { - log::error!( - "Invalidation fault detected: segment={}, FSTS=0x{:x}, fault_flags={:?}", - segment_number, - fsts, - fault_flags - ); - - // Clear fault status bits (W1C) - registers::FSTS.write(fsts, vt_d_base.as_usize()); - - return Err(VtdError::FaultStatus(fsts)); - } } log::warn!( - "Invalidation wait timeout: segment={}, ICS=0x{:x}", + "Vt-d: Invalidation wait timeout: segment = {}, IQH = 0x{:x}, IQT = 0x{:x}", segment_number, - registers::ICS.read(vt_d_base.as_usize()) + registers::IQH.read(vt_d_base.as_usize()), + registers::IQT.read(vt_d_base.as_usize()) ); Err(VtdError::Timeout) } /// Invalidate interrupt entry cache for a specific index +#[allow(dead_code)] fn invalidate_interrupt_entry( segment_number: usize, vtd_addrs: &VtdAddrs, @@ -699,10 +727,10 @@ fn invalidate_interrupt_entry( push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; // Wait for completion - wait_invalidation_complete(segment_number, vt_d_base)?; + wait_invalidation_complete(segment_number, vtd_addrs)?; log::debug!( - "Invalidated interrupt entry: segment={}, index={}", + "Vt-d: Invalidated interrupt entry: segment = {}, index = {}", segment_number, interrupt_index ); @@ -725,15 +753,11 @@ fn invalidate_all_interrupt_entries( // Push descriptor to queue push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; - // Create and push Invalidation Wait descriptor - let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; - // Wait for completion - wait_invalidation_complete(segment_number, vt_d_base)?; + wait_invalidation_complete(segment_number, vtd_addrs)?; log::info!( - "Invalidated all interrupt entries: segment={}", + "Vt-d: Invalidated all interrupt entries: segment = {}", segment_number ); From f878937853bc36a03dd2a0db0b42cacba62c4e35 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 17:47:52 +0900 Subject: [PATCH 10/16] fix(vt-d): print some registers when invalidating cache Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 9e9d751aa..76f8f1dc6 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -347,11 +347,11 @@ pub unsafe fn init_interrupt_remap( let mut node = MCSNode::new(); let vtd_units = iommu_info.vtd_units.lock(&mut node); - if let Some(vtd_addrs) = vtd_units.front() { + for vtd_addrs in vtd_units.iter() { let vt_d_base = vtd_addrs.vt_d_base; let ecap = registers::EXTENDED_CAPABILITY.read(vt_d_base.as_usize()); - // Check if ESIRTPS is supported (bit 2) + // Check if ESIRTPS is supported (bit 62) if (ecap & registers::ECAP_ESIRTPS) == 0 { // ESIRTPS not supported, need to perform global invalidation log::info!( @@ -730,9 +730,13 @@ fn invalidate_interrupt_entry( wait_invalidation_complete(segment_number, vtd_addrs)?; log::debug!( - "Vt-d: Invalidated interrupt entry: segment = {}, index = {}", + "Vt-d: Invalidated interrupt entry: segment = {}, index = {}, vtd_base = 0x{:x}, iq_base = 0x{:x}, IQH = 0x{:x}, IQT = 0x{:x}", segment_number, - interrupt_index + interrupt_index, + vt_d_base.as_usize(), + iq_base.as_usize(), + registers::IQH.read(vt_d_base.as_usize()), + registers::IQT.read(vt_d_base.as_usize()) ); Ok(()) @@ -757,8 +761,12 @@ fn invalidate_all_interrupt_entries( wait_invalidation_complete(segment_number, vtd_addrs)?; log::info!( - "Vt-d: Invalidated all interrupt entries: segment = {}", - segment_number + "Vt-d: Invalidated all interrupt entries: segment = {}, vtd_base = 0x{:x}, iq_base = 0x{:x}, IQH = 0x{:x}, IQT = 0x{:x}", + segment_number, + vt_d_base.as_usize(), + iq_base.as_usize(), + registers::IQH.read(vt_d_base.as_usize()), + registers::IQT.read(vt_d_base.as_usize()) ); Ok(()) From b37bf6e88a15e63d13343b7a4a9b592df137bc88 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 18:20:23 +0900 Subject: [PATCH 11/16] fix(vt-d): fix granuality Signed-off-by: Yuuki Takano --- awkernel_lib/src/arch/x86_64/interrupt_remap.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 76f8f1dc6..7f3f82a0a 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -716,8 +716,8 @@ fn invalidate_interrupt_entry( let iq_base = vtd_addrs.iq_base; // Create Interrupt Entry Cache Invalidation descriptor - // Granularity = 0 (global), index_mask = 0 (specific index) - let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(interrupt_index, 0, 0); + // Granularity = 1 (index-selective), index_mask = 0 (specific index) + let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(interrupt_index, 0, 1); // Push descriptor to queue push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; @@ -751,12 +751,11 @@ fn invalidate_all_interrupt_entries( let iq_base = vtd_addrs.iq_base; // Create global Interrupt Entry Cache Invalidation descriptor - // Granularity = 1 (global) - let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(0, 0, 1); + // Granularity = 0 (global) + let desc = InvalidationDescriptor::interrupt_entry_cache_invalidation(0, 0, 0); // Push descriptor to queue push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; - // Wait for completion wait_invalidation_complete(segment_number, vtd_addrs)?; From 4a85e7da5f9d7d1a1fbe0626c09690613621240f Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 18:25:07 +0900 Subject: [PATCH 12/16] fix(vt-d): check the fault status register Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 7f3f82a0a..9e35f9cf6 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -689,6 +689,26 @@ fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdAddrs) -> Re push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; for _ in 0..10000 { + // Check for fault status before checking completion + let fsts = registers::FSTS.read(vt_d_base.as_usize()); + let fault_flags = registers::FaultStatus::from_bits_truncate(fsts); + if fault_flags.intersects( + registers::FaultStatus::IQE | registers::FaultStatus::ITE | registers::FaultStatus::ICE, + ) { + log::error!( + "Vt-d: Invalidation fault detected: segment = {}, FSTS = 0x{:x}, fault_flags = {:?}", + segment_number, + fsts, + fault_flags + ); + + // Clear fault status bits (W1C) + registers::FSTS.write(fsts, vt_d_base.as_usize()); + + return Err(VtdError::FaultStatus(fsts)); + } + + // Check for completion core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); if status[0] != 0 { return Ok(()); From b2454389677e5240941d8fc2549bba42f09f0637 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Thu, 8 Jan 2026 19:05:58 +0900 Subject: [PATCH 13/16] fix(idt): wait 0 for IRTPS first Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 9e35f9cf6..5f7ab99e6 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -438,6 +438,24 @@ fn wait_toggle_then_set( enable: bool, ) -> Result<(), VtdError> { if enable { + // For IRTP, wait until 0 first + // + // IRTPS: + // This field is cleared by hardware when software sets the SIRTP field in the + // Global Command register. This field is Set by hardware when hardware + // completes the ‘Set Interrupt Remap Table Pointer’ operation using the + // value provided in the Interrupt Remapping Table Address register. + if status_bit.contains(registers::GlobalCommandStatus::IRTP) { + for _ in 0..1000 { + if !registers::GLOBAL_STATUS + .read(vt_d_base.as_usize()) + .contains(registers::GlobalCommandStatus::IRTP) + { + break; + } + } + } + // wait until 1 for _ in 0..1000 { if registers::GLOBAL_STATUS From d298fb8003ea2c4006c17e9d562e198fdc96ad8b Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Fri, 9 Jan 2026 13:08:49 +0900 Subject: [PATCH 14/16] fix(vt-d): invalidate cache when allocating and dealocating IRTE Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 102 ++++++++++++++---- 1 file changed, 82 insertions(+), 20 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 5f7ab99e6..e3c58e05e 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -1,5 +1,7 @@ //! Interrupt remapping of Vt-d. +use core::sync::atomic::AtomicUsize; + use acpi::AcpiTables; use alloc::{collections::linked_list::LinkedList, format, string::String}; @@ -18,6 +20,11 @@ use super::acpi::AcpiMapper; const TABLE_SIZE: usize = 256 * 4096; // 1MiB const TABLE_ENTRY_NUM: usize = TABLE_SIZE / 16; +static VTD_UNIT_IDX: AtomicUsize = AtomicUsize::new(0); + +static DMA_POOL: [Mutex>>; 32] = + array![_ => Mutex::new(None); 32]; + /// Error type for Vt-d interrupt remapping initialization. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VtdError { @@ -184,14 +191,15 @@ static IOMMU: [IommuInfo; 32] = array![x => #[allow(dead_code)] // TODO: remove this struct IommuInfo { segment_number: usize, - vtd_units: Mutex>, + vtd_units: Mutex>, interrupt_remapping: Mutex>, } #[allow(dead_code)] // TODO: remove this -struct VtdAddrs { +struct VtdUnit { vt_d_base: VirtAddr, iq_base: VirtAddr, // Cache Invalidation Queue base address + index: usize, } #[derive(Debug)] @@ -208,6 +216,12 @@ impl RemapInfo { impl Drop for RemapInfo { fn drop(&mut self) { + log::debug!( + "Vt-d: Deallocating interrupt remapping entry: segment = {}, entry_id = {}", + self.segment_number, + self.entry_id + ); + let mut node = MCSNode::new(); let mut table = IOMMU[self.segment_number] .interrupt_remapping @@ -215,6 +229,21 @@ impl Drop for RemapInfo { if let Some(table) = table.as_mut() { table.deallocate_entry(self.entry_id); + + core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + + let mut node = MCSNode::new(); + let vtd_units = IOMMU[self.segment_number].vtd_units.lock(&mut node); + for vtd_addrs in vtd_units.iter() { + if let Err(e) = invalidate_all_interrupt_entries(self.segment_number, vtd_addrs) { + log::error!( + "Vt-d: Failed to invalidate interrupt entry after deallocation: segment = {}, entry_id = {}, error = {:?}", + self.segment_number, + self.entry_id, + e + ); + } + } } } } @@ -515,14 +544,19 @@ fn set_irta( // Enable IRTP (Interrupt Remapping Table Pointer) update_global_command(vt_d_base, registers::GlobalCommandStatus::IRTP, true)?; - // Enable IRE (Interrupt Remapping Enable) - update_global_command(vt_d_base, registers::GlobalCommandStatus::IRE, true)?; - // Initialize Invalidation Queue let iq_base = init_invalidation_queue(segment_number, vt_d_base)?; + // Enable IRE (Interrupt Remapping Enable) + update_global_command(vt_d_base, registers::GlobalCommandStatus::IRE, true)?; + // Store VT-d unit info - let vtd_addrs = VtdAddrs { vt_d_base, iq_base }; + let index = VTD_UNIT_IDX.fetch_add(1, core::sync::atomic::Ordering::SeqCst); + let vtd_addrs = VtdUnit { + vt_d_base, + iq_base, + index, + }; let mut node = MCSNode::new(); IOMMU[segment_number] @@ -689,16 +723,29 @@ fn push_invalidation_descriptor( } /// Wait for invalidation completion by checking ICS or using wait descriptor -fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdAddrs) -> Result<(), VtdError> { +fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdUnit) -> Result<(), VtdError> { let vt_d_base = vtd_addrs.vt_d_base; let iq_base = vtd_addrs.iq_base; // Create and push Invalidation Wait descriptor - let mut dma_pool = DMAPool::<[u32; PAGESIZE / 4]>::new(segment_number, 1) - .ok_or(VtdError::DmaAllocationFailed)?; + let mut node = MCSNode::new(); + let mut dma_pool = DMA_POOL[segment_number].lock(&mut node); + + if dma_pool.is_none() { + let pool = DMAPool::<[u32; PAGESIZE / 4]>::new(segment_number, 1) + .ok_or(VtdError::DmaAllocationFailed)?; + *dma_pool = Some(pool); + } - let status_addr = dma_pool.get_phy_addr().as_usize() as u64; - let status = dma_pool.as_mut(); + let status_addr; + let status; + + if let Some(dma_pool) = dma_pool.as_mut() { + status_addr = dma_pool.get_phy_addr().as_usize() as u64; + status = dma_pool.as_mut(); + } else { + return Err(VtdError::DmaAllocationFailed); + } status[0] = 0; core::sync::atomic::fence(core::sync::atomic::Ordering::Release); @@ -744,10 +791,9 @@ fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdAddrs) -> Re } /// Invalidate interrupt entry cache for a specific index -#[allow(dead_code)] fn invalidate_interrupt_entry( segment_number: usize, - vtd_addrs: &VtdAddrs, + vtd_addrs: &VtdUnit, interrupt_index: u16, ) -> Result<(), VtdError> { let vt_d_base = vtd_addrs.vt_d_base; @@ -760,14 +806,10 @@ fn invalidate_interrupt_entry( // Push descriptor to queue push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; - // Create and push Invalidation Wait descriptor - let wait_desc = InvalidationDescriptor::invalidation_wait(false, 0, 0); - push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; - // Wait for completion wait_invalidation_complete(segment_number, vtd_addrs)?; - log::debug!( + log::trace!( "Vt-d: Invalidated interrupt entry: segment = {}, index = {}, vtd_base = 0x{:x}, iq_base = 0x{:x}, IQH = 0x{:x}, IQT = 0x{:x}", segment_number, interrupt_index, @@ -783,7 +825,7 @@ fn invalidate_interrupt_entry( /// Invalidate all interrupt entries (global invalidation) fn invalidate_all_interrupt_entries( segment_number: usize, - vtd_addrs: &VtdAddrs, + vtd_addrs: &VtdUnit, ) -> Result<(), VtdError> { let vt_d_base = vtd_addrs.vt_d_base; let iq_base = vtd_addrs.iq_base; @@ -794,6 +836,7 @@ fn invalidate_all_interrupt_entries( // Push descriptor to queue push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; + // Wait for completion wait_invalidation_complete(segment_number, vtd_addrs)?; @@ -820,7 +863,26 @@ pub fn allocate_remapping_entry( let mut table = IOMMU[segment_number].interrupt_remapping.lock(&mut node); if let Some(table) = table.as_mut() { - table.allocate_entry(dest_apic_id, irq, level_trigger, lowest_priority) + let remap_info = table.allocate_entry(dest_apic_id, irq, level_trigger, lowest_priority)?; + + core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + + let mut node = MCSNode::new(); + let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); + for vtd_addrs in vtd_units.iter() { + if let Err(e) = + invalidate_interrupt_entry(segment_number, vtd_addrs, remap_info.entry_id as u16) + { + log::error!( + "Vt-d: Failed to invalidate interrupt entry after allocation: segment = {}, entry_id = {}, error = {:?}", + segment_number, + remap_info.entry_id, + e + ); + } + } + + Some(remap_info) } else { None } From a9f838861040971eecc626a0cf05a3413bb79521 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 13 Jan 2026 16:34:18 +0900 Subject: [PATCH 15/16] fix(vt-d): minor fix Signed-off-by: Yuuki Takano --- .../src/arch/x86_64/interrupt_remap.rs | 115 +++++++++++++----- 1 file changed, 86 insertions(+), 29 deletions(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index 97a2bf39e..cc278ef11 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -113,6 +113,8 @@ mod registers { } } + pub const ICS_IWC: u32 = 1 << 0; // Invalidation Write Complete + bitflags! { #[derive(Debug, Clone, Copy)] pub struct FaultStatus: u32 { @@ -122,8 +124,6 @@ mod registers { } } - pub const ICS_IWC: u32 = 1 << 0; // Invalidation Write Complete - // Extended Capability Register bits pub const ECAP_ESIRTPS: u64 = 1 << 62; // Enhanced Set Interrupt Root Table Pointer Support @@ -199,14 +199,12 @@ static IOMMU: [IommuInfo; 32] = array![x => interrupt_remapping: Mutex::new(None), }; 32]; -#[allow(dead_code)] // TODO: remove this struct IommuInfo { segment_number: usize, vtd_units: Mutex>, interrupt_remapping: Mutex>, } -#[allow(dead_code)] // TODO: remove this struct VtdUnit { vt_d_base: VirtAddr, iq_base: VirtAddr, // Cache Invalidation Queue base address @@ -219,6 +217,48 @@ pub struct RemapInfo { segment_number: usize, } +impl IommuInfo { + fn dump(&self) -> Option { + let mut result = String::new(); + + { + let mut node = MCSNode::new(); + let vtd_units = self.vtd_units.lock(&mut node); + + if vtd_units.is_empty() { + return None; + } + + result = format!("VT-d Segment {}:", self.segment_number); + + for (i, vtd_unit) in vtd_units.iter().enumerate() { + result = format!( + "{result}\r\n VT-d Unit [{i}]: VT-d Base = 0x{:x}, IQ Base = 0x{:x}, Index = {}", + vtd_unit.vt_d_base.as_usize(), + vtd_unit.iq_base.as_usize(), + vtd_unit.index + ); + } + } + + { + let mut node = MCSNode::new(); + let interrupt_remapping = self.interrupt_remapping.lock(&mut node); + if let Some(table) = interrupt_remapping.as_ref() { + result = format!( + "{result}\r\n Interrupt Remapping Table: Base = 0x{:x}, is_x2apic = {}", + table.table_base.as_usize(), + table.is_x2apic + ); + } else { + result = format!("{result}\r\n Interrupt Remapping Table: Not initialized"); + } + } + + Some(result) + } +} + impl RemapInfo { pub fn get_entry_id(&self) -> usize { self.entry_id @@ -245,8 +285,8 @@ impl Drop for RemapInfo { let mut node = MCSNode::new(); let vtd_units = IOMMU[self.segment_number].vtd_units.lock(&mut node); - for vtd_addrs in vtd_units.iter() { - if let Err(e) = invalidate_all_interrupt_entries(self.segment_number, vtd_addrs) { + for vtd_unit in vtd_units.iter() { + if let Err(e) = invalidate_all_interrupt_entries(self.segment_number, vtd_unit) { log::error!( "Vt-d: Failed to invalidate interrupt entry after deallocation: segment = {}, entry_id = {}, error = {:?}", self.segment_number, @@ -399,8 +439,8 @@ pub unsafe fn init_interrupt_remap( let mut node = MCSNode::new(); let vtd_units = iommu_info.vtd_units.lock(&mut node); - for vtd_addrs in vtd_units.iter() { - let vt_d_base = vtd_addrs.vt_d_base; + for vtd_unit in vtd_units.iter() { + let vt_d_base = vtd_unit.vt_d_base; let ecap = registers::EXTENDED_CAPABILITY.read(vt_d_base.as_usize()); // Check if ESIRTPS is supported (bit 62) @@ -412,8 +452,8 @@ pub unsafe fn init_interrupt_remap( ecap ); - // Perform global invalidation using the vtd_addrs reference - invalidate_all_interrupt_entries(segment_number, vtd_addrs)?; + // Perform global invalidation using the vtd_unit reference + invalidate_all_interrupt_entries(segment_number, vtd_unit)?; } else { log::debug!( "VT-d segment {}: ESIRTPS supported (ECAP=0x{:x}), skipping global invalidation", @@ -424,6 +464,8 @@ pub unsafe fn init_interrupt_remap( } } + dump_iommu_info(); + Ok(()) } @@ -590,7 +632,7 @@ fn enable_interrupt_remapping( // Store VT-d unit info let index = VTD_UNIT_IDX.fetch_add(1, core::sync::atomic::Ordering::SeqCst); - let vtd_addrs = VtdUnit { + let vtd_unit = VtdUnit { vt_d_base, iq_base, index, @@ -600,7 +642,7 @@ fn enable_interrupt_remapping( IOMMU[segment_number] .vtd_units .lock(&mut node) - .push_back(vtd_addrs); + .push_back(vtd_unit); // Re-read status after queue initialization let status = registers::GLOBAL_STATUS.read(vt_d_base.as_usize()); @@ -761,9 +803,9 @@ fn push_invalidation_descriptor( } /// Wait for invalidation completion by checking ICS or using wait descriptor -fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdUnit) -> Result<(), VtdError> { - let vt_d_base = vtd_addrs.vt_d_base; - let iq_base = vtd_addrs.iq_base; +fn wait_invalidation_complete(segment_number: usize, vtd_unit: &VtdUnit) -> Result<(), VtdError> { + let vt_d_base = vtd_unit.vt_d_base; + let iq_base = vtd_unit.iq_base; // Create and push Invalidation Wait descriptor let mut node = MCSNode::new(); @@ -775,19 +817,22 @@ fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdUnit) -> Res *dma_pool = Some(pool); } - let status_addr; - let status; + let status_addr; // Physical address of the status word + let status; // Mutable reference to the status word if let Some(dma_pool) = dma_pool.as_mut() { - status_addr = dma_pool.get_phy_addr().as_usize() as u64; + status_addr = (dma_pool.get_phy_addr().as_usize() + vtd_unit.index * 4) as u64; status = dma_pool.as_mut(); } else { return Err(VtdError::DmaAllocationFailed); } - status[0] = 0; + // Initialize status word to 0 + status[vtd_unit.index] = 0; core::sync::atomic::fence(core::sync::atomic::Ordering::Release); + // Create Invalidation Wait descriptor + // When the invalidation is complete, hardware writes 1 to status_addr. let wait_desc = InvalidationDescriptor::invalidation_wait(true, 1, status_addr); push_invalidation_descriptor(segment_number, vt_d_base, iq_base, wait_desc)?; @@ -831,11 +876,11 @@ fn wait_invalidation_complete(segment_number: usize, vtd_addrs: &VtdUnit) -> Res /// Invalidate interrupt entry cache for a specific index fn invalidate_interrupt_entry( segment_number: usize, - vtd_addrs: &VtdUnit, + vtd_unit: &VtdUnit, interrupt_index: u16, ) -> Result<(), VtdError> { - let vt_d_base = vtd_addrs.vt_d_base; - let iq_base = vtd_addrs.iq_base; + let vt_d_base = vtd_unit.vt_d_base; + let iq_base = vtd_unit.iq_base; // Create Interrupt Entry Cache Invalidation descriptor // Granularity = 1 (index-selective), index_mask = 0 (specific index) @@ -845,7 +890,7 @@ fn invalidate_interrupt_entry( push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; // Wait for completion - wait_invalidation_complete(segment_number, vtd_addrs)?; + wait_invalidation_complete(segment_number, vtd_unit)?; log::trace!( "Vt-d: Invalidated interrupt entry: segment = {}, index = {}, vtd_base = 0x{:x}, iq_base = 0x{:x}, IQH = 0x{:x}, IQT = 0x{:x}", @@ -863,10 +908,10 @@ fn invalidate_interrupt_entry( /// Invalidate all interrupt entries (global invalidation) fn invalidate_all_interrupt_entries( segment_number: usize, - vtd_addrs: &VtdUnit, + vtd_unit: &VtdUnit, ) -> Result<(), VtdError> { - let vt_d_base = vtd_addrs.vt_d_base; - let iq_base = vtd_addrs.iq_base; + let vt_d_base = vtd_unit.vt_d_base; + let iq_base = vtd_unit.iq_base; // Create global Interrupt Entry Cache Invalidation descriptor // Granularity = 0 (global) @@ -876,7 +921,7 @@ fn invalidate_all_interrupt_entries( push_invalidation_descriptor(segment_number, vt_d_base, iq_base, desc)?; // Wait for completion - wait_invalidation_complete(segment_number, vtd_addrs)?; + wait_invalidation_complete(segment_number, vtd_unit)?; log::info!( "Vt-d: Invalidated all interrupt entries: segment = {}, vtd_base = 0x{:x}, iq_base = 0x{:x}, IQH = 0x{:x}, IQT = 0x{:x}", @@ -907,9 +952,9 @@ pub fn allocate_remapping_entry( let mut node = MCSNode::new(); let vtd_units = IOMMU[segment_number].vtd_units.lock(&mut node); - for vtd_addrs in vtd_units.iter() { + for vtd_unit in vtd_units.iter() { if let Err(e) = - invalidate_interrupt_entry(segment_number, vtd_addrs, remap_info.entry_id as u16) + invalidate_interrupt_entry(segment_number, vtd_unit, remap_info.entry_id as u16) { log::error!( "Vt-d: Failed to invalidate interrupt entry after allocation: segment = {}, entry_id = {}, error = {:?}", @@ -925,3 +970,15 @@ pub fn allocate_remapping_entry( None } } + +fn dump_iommu_info() { + let mut result = String::new(); + + for iommu_info in IOMMU.iter() { + if let Some(dump) = iommu_info.dump() { + result = format!("{result}\r\n{dump}"); + } + } + + log::info!("Vt-d IOMMU Info: {result}"); +} From e1679c426e9391252c642b80cded8d15921b4576 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 13 Jan 2026 16:44:58 +0900 Subject: [PATCH 16/16] minor fix Signed-off-by: Yuuki Takano --- awkernel_lib/src/arch/x86_64/interrupt_remap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs index cc278ef11..9159feb7a 100644 --- a/awkernel_lib/src/arch/x86_64/interrupt_remap.rs +++ b/awkernel_lib/src/arch/x86_64/interrupt_remap.rs @@ -113,7 +113,7 @@ mod registers { } } - pub const ICS_IWC: u32 = 1 << 0; // Invalidation Write Complete + pub const ICS_IWC: u32 = 1 << 0; // Invalidation Wait Descriptor Complete bitflags! { #[derive(Debug, Clone, Copy)]