diff --git a/aarch64/src/devcons.rs b/aarch64/src/devcons.rs index 1c4dcd4..6b1fd55 100644 --- a/aarch64/src/devcons.rs +++ b/aarch64/src/devcons.rs @@ -3,7 +3,7 @@ use crate::uartmini::MiniUart; use core::cell::SyncUnsafeCell; use core::mem::MaybeUninit; -use port::devcons::Console; +use port::devcons::{Console, IprintOps, Uart}; use port::fdt::DeviceTree; #[cfg(not(test))] use port::println; @@ -25,6 +25,19 @@ use port::println; // - UART2 PL011 // - UART3 PL011 +static UART: SyncUnsafeCell> = SyncUnsafeCell::new(MaybeUninit::uninit()); + +static IPRINT_OPS: IprintOps = IprintOps { putb: iputb }; + +/// Direct polled write for iprint, bypassing the console lock. +/// `MiniUart::putb` needs only a shared reference, so this can safely +/// alias the reference held by the console. +fn iputb(b: u8) { + // Safety: IPRINT_OPS is only registered once UART is initialised. + let uart = unsafe { (*UART.get()).assume_init_ref() }; + uart.putb(b); +} + pub fn init(dt: &DeviceTree) { Console::set_uart(|| { let uart = MiniUart::new_with_map_ranges(dt); @@ -35,12 +48,11 @@ pub fn init(dt: &DeviceTree) { Ok(uart) => { uart.init(); - static UART: SyncUnsafeCell> = - SyncUnsafeCell::new(MaybeUninit::uninit()); unsafe { let cons = &mut *UART.get(); cons.write(uart); - Ok(cons.assume_init_mut()) + port::devcons::set_iprint_ops(&IPRINT_OPS); + Ok(cons.assume_init_ref()) } } Err(msg) => { diff --git a/aarch64/src/gic.rs b/aarch64/src/gic.rs new file mode 100644 index 0000000..a26d7a2 --- /dev/null +++ b/aarch64/src/gic.rs @@ -0,0 +1,278 @@ +//! Generic Interrupt Controller driver. +//! +//! Currently supports GIC-400/GICv2. +//! Initialises the distributor and CPU interface, provides IRQ enable/disable +//! and priority management. + +use core::fmt; + +use crate::deviceutil::map_device_register; +use crate::io::{read_reg, write_reg}; +use crate::vm; +use port::Result; +use port::fdt::DeviceTree; +use port::irq::IrqGuard; +use port::mcslock::{Lock, LockNode}; +use port::mem::{PhysRange, VirtRange}; + +use bitstruct::bitstruct; + +#[cfg(not(test))] +use port::println; + +const GICC_CTLR: usize = 0x0000; +const GICC_PMR: usize = 0x0004; +const GICC_IAR: usize = 0x000c; +const GICC_EOIR: usize = 0x0010; +const GICC_IIDR: usize = 0x00fc; // CPU Interface Identification Register + +const GICD_CTLR: usize = 0x0000; +const GICD_ISENABLER: usize = 0x0100; // Set-enable registers (0x100-0x17c) +const GICD_ICENABLER: usize = 0x0180; // Clear-enable registers (0x180-0x1fc) + +// INTIDs 1020..=1023 are special (1023 = spurious: no pending interrupt). +const INTID_SPECIAL_START: u16 = 1020; + +// EL1 physical timer PPI. DT PPI numbers map to INTIDs as 16 + n: +// secure phys timer is DT PPI 13 → INTID 29, non-secure phys is +// DT PPI 14 → INTID 30. Which one CNTP_* raises depends on the +// security state we boot in. +pub const TIMER_INTID: u16 = 30; + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GiccIidr(pub u32) { + pub implementer: u32 = 0..12; + pub revision: u16 = 12..16; + pub arch_version: u8 = 16..20; + pub product_id: u16 = 20..32; + } +} + +impl fmt::Debug for GiccIidr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GiccIidr") + .field("implementer", &format_args!("{:#x}", self.implementer())) + .field("revision", &format_args!("{}", self.revision())) + .field("arch_version", &format_args!("{}", self.arch_version())) + .field("product_id", &format_args!("{:#x}", self.product_id())) + .finish() + } +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GiccIar(pub u32) { + pub int_id: u16 = 0..10; + pub cpu_id: u8 = 10..13; + } +} + +impl fmt::Debug for GiccIar { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GiccIar") + .field("int_id", &format_args!("{}", self.int_id())) + .field("cpu_id", &format_args!("{}", self.cpu_id())) + .finish() + } +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GicdTyper(pub u32) { + pub it_lines_number: u8 = 0..5; + pub cpu_number: u8 = 5..8; + pub security_extn: u8 = 10..11; + pub lspi: u16 = 11..16; + } +} + +impl fmt::Debug for GicdTyper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GicdTyper") + .field("it_lines_number", &format_args!("{:#}", self.it_lines_number())) + .field("cpu_number", &format_args!("{}", self.cpu_number())) + .field("security_extn", &format_args!("{}", self.security_extn())) + .field("lspi", &format_args!("{}", self.lspi())) + .finish() + } +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GicdCtlr(pub u32) { + pub enable: bool = 0..1; + } +} + +impl From for u32 { + fn from(r: GicdCtlr) -> u32 { + r.0 + } +} + +impl fmt::Debug for GicdCtlr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GicdCtlr").field("enable", &format_args!("{:#}", self.enable())).finish() + } +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GiccCtlr(pub u32) { + pub enable: bool = 0..1; + } +} + +impl From for u32 { + fn from(r: GiccCtlr) -> u32 { + r.0 + } +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct GiccPmr(pub u32) { + pub priority: u8 = 0..8; + } +} + +impl From for u32 { + fn from(r: GiccPmr) -> u32 { + r.0 + } +} + +static GIC: Lock> = Lock::new("gic", None); + +pub fn init(dt: &DeviceTree) { + match Gic::new(dt) { + Ok(gic) => { + let _irq = IrqGuard::new(); + let node = LockNode::new(); + let mut locked_gic = GIC.lock(&node); + *locked_gic = Some(gic); + } + Err(msg) => { + println!("can't initialise gic: {:?}", msg); + } + } +} + +struct Gic { + gicc_virtrange: VirtRange, + gicd_virtrange: VirtRange, +} + +impl Gic { + pub fn new(dt: &DeviceTree) -> Result { + let (gicc_virtrange, gicd_virtrange) = find_gicc_gicd_virtranges(dt, "arm,gic-400") + .or_else(|_| find_gicc_gicd_virtranges(dt, "arm,cortex-a15-gic"))?; + + let gicc_iidr = GiccIidr(read_reg(&gicc_virtrange, GICC_IIDR)); + if gicc_iidr.arch_version() == 1 { + return Err("gic v1 unsupported"); + } + + let mut gic = Gic { gicc_virtrange, gicd_virtrange }; + + // Enable distributor + write_reg(&gic.gicd_virtrange, GICD_CTLR, GicdCtlr(0).with_enable(true).into()); + // Enable timer interrupt + gic.enable_interrupt(TIMER_INTID); + // Admit all priorities (lower value = higher priority; 0xff is + // the lowest threshold, masking nothing) + write_reg(&gic.gicc_virtrange, GICC_PMR, GiccPmr(0).with_priority(0xff).into()); + // Enable CPU interface + write_reg(&gic.gicc_virtrange, GICC_CTLR, GiccCtlr(0).with_enable(true).into()); + + println!("gic: initialised"); + + Ok(gic) + } + + /// Acknowledge the highest-priority pending interrupt by reading + /// GICC_IAR. Returns the raw IAR value (INTID in bits 9:0), or None + /// if no interrupt is pending. The returned value must be passed to + /// `eoi` after the interrupt source has been handled. + fn try_ack_interrupt(&mut self) -> Option { + let iar = GiccIar(read_reg(&self.gicc_virtrange, GICC_IAR)); + if iar.int_id() >= INTID_SPECIAL_START { None } else { Some(iar) } + } + + /// Signal end of interrupt by writing the raw IAR value back to GICC_EOIR. + fn end_interrupt(&mut self, iar: GiccIar) { + write_reg(&self.gicc_virtrange, GICC_EOIR, iar.0); + } + + /// Enable delivery of an interrupt at the distributor. + fn enable_interrupt(&mut self, intid: u16) { + let n = intid as usize; + write_reg(&self.gicd_virtrange, GICD_ISENABLER + 4 * (n / 32), 1 << (n % 32)); + } + + /// Stop delivery of an interrupt at the distributor. + fn disable_interrupt(&mut self, intid: u16) { + let n = intid as usize; + write_reg(&self.gicd_virtrange, GICD_ICENABLER + 4 * (n / 32), 1 << (n % 32)); + } +} + +fn find_gicc_gicd_virtranges(dt: &DeviceTree, id: &'static str) -> Result<(VirtRange, VirtRange)> { + // The GICD reg is first (index 0), GICC is second (index 1) + if let Some(gic_node) = dt.find_compatible(id).next() { + let gicc_virtrange = dt + .property_translated_reg_iter(gic_node) + .nth(1) + .and_then(|reg| reg.regblock()) + .map(|reg| PhysRange::from(®)) + .map(|physrange| map_device_register("gicc", physrange, vm::PageSize::Page4K)) + .unwrap_or(Err("can't get gicc regblock from devicetree"))?; + + let gicd_virtrange = dt + .property_translated_reg_iter(gic_node) + .next() + .and_then(|reg| reg.regblock()) + .map(|reg| PhysRange::from(®)) + .map(|physrange| map_device_register("gicd", physrange, vm::PageSize::Page4K)) + .unwrap_or(Err("can't get gicd regblock from devicetree"))?; + + Ok((gicc_virtrange, gicd_virtrange)) + } else { + Err("Couldn't parse gic node in devicetree") + } +} + +/// Called from the trap handler to acknowledge a pending GIC interrupt. +/// Returns the raw IAR value, or None if no interrupt was pending +/// (spurious) or the GIC is not initialised. The caller must handle the +/// interrupt (deasserting its source — the timer PPI is level-triggered) +/// and then pass the value to `eoi`. EOI before deassertion would +/// immediately re-raise the interrupt. +pub fn try_ack_interrupt() -> Option { + let _irq = IrqGuard::new(); + let node = LockNode::new(); + let mut guard = GIC.lock(&node); + guard.as_mut().and_then(|gic| gic.try_ack_interrupt()) +} + +/// Called from the trap handler for an IRQ nothing claims. +pub fn disable_interrupt(intid: u16) { + let _irq = IrqGuard::new(); + let node = LockNode::new(); + let mut guard = GIC.lock(&node); + if let Some(gic) = guard.as_mut() { + gic.disable_interrupt(intid); + } +} + +/// Called from the trap handler to signal end-of-interrupt for a value +/// previously returned by `ack`. +pub fn end_interrupt(iar: GiccIar) { + let _irq = IrqGuard::new(); + let node = LockNode::new(); + let mut guard = GIC.lock(&node); + if let Some(gic) = guard.as_mut() { + gic.end_interrupt(iar); + } +} diff --git a/aarch64/src/irq.rs b/aarch64/src/irq.rs new file mode 100644 index 0000000..866392c --- /dev/null +++ b/aarch64/src/irq.rs @@ -0,0 +1,36 @@ +//! DAIF-based implementation of the portable interrupt masking hooks. + +use port::irq::IrqOps; + +static IRQ_OPS: IrqOps = IrqOps { mask: mask_irqs, restore: restore_irqs }; + +/// Register DAIF masking with `port::irq`. Must be called before +/// interrupts are enabled. +pub fn init() { + port::irq::set_ops(&IRQ_OPS); +} + +/// Mask IRQs on this core, returning the previous DAIF state. +fn mask_irqs() -> u64 { + let daif: u64; + unsafe { + core::arch::asm!( + "mrs {daif}, daif", + "msr daifset, #2", + daif = out(reg) daif, + options(nostack, preserves_flags) + ); + } + daif +} + +/// Restore a DAIF state previously returned by `mask_irqs`. +fn restore_irqs(daif: u64) { + unsafe { + core::arch::asm!( + "msr daif, {daif}", + daif = in(reg) daif, + options(nostack, preserves_flags) + ); + } +} diff --git a/aarch64/src/mailbox.rs b/aarch64/src/mailbox.rs index fcafef7..ad3e229 100644 --- a/aarch64/src/mailbox.rs +++ b/aarch64/src/mailbox.rs @@ -38,21 +38,23 @@ pub fn init(dt: &DeviceTree) { /// https://developer.arm.com/documentation/ddi0306/b/CHDGHAIG /// https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface struct Mailbox { - pub mbox_virtrange: VirtRange, - req_buffer_va: VirtRange, - req_buffer_pa: PhysRange, + mbox_virtrange: VirtRange, + req_buf_virtrange: VirtRange, + req_buf_physrange: PhysRange, } impl Mailbox { fn new(dt: &DeviceTree) -> Result { // Allocate a page of device memory for the mailbox request/response buffer // TODO Split this into multiple buffers to allow parallel requests. - let (req_buffer_va, req_buffer_pa) = + let (req_buf_virtrange, req_buf_physrange) = deviceutil::alloc_device_page("mailboxbuf", vm::PageSize::Page4K)?; let mbox_physrange = Self::find_mbox_physrange(dt)?; let mbox = match map_device_register("mailbox", mbox_physrange, vm::PageSize::Page4K) { - Ok(mbox_virtrange) => Ok(Mailbox { mbox_virtrange, req_buffer_va, req_buffer_pa }), + Ok(mbox_virtrange) => { + Ok(Mailbox { mbox_virtrange, req_buf_virtrange, req_buf_physrange }) + } Err(msg) => { println!("can't map mailbox {:?}", msg); Err("can't create mailbox") @@ -78,7 +80,7 @@ impl Mailbox { // Write the request address combined with the channel to the write register let channel = ChannelId::ArmToVc as u32; - let uart_mbox_u32 = self.req_buffer_pa.start.addr() as u32; + let uart_mbox_u32 = self.req_buf_physrange.start.addr() as u32; let r = (uart_mbox_u32 & !0xF) | channel; write_reg(&self.mbox_virtrange, MBOX_WRITE, r); @@ -146,7 +148,7 @@ where .as_mut() .map(|mb| { let msg = unsafe { - let page_va_ptr = mb.req_buffer_va.start as u64 as *mut MessageWithTags; + let page_va_ptr = mb.req_buf_virtrange.start as u64 as *mut MessageWithTags; core::intrinsics::volatile_set_memory(page_va_ptr, 0, 1); let msg = NonNull::new_unchecked(page_va_ptr).as_mut(); msg.request.size = size; diff --git a/aarch64/src/main.rs b/aarch64/src/main.rs index a3eaa29..4168c42 100644 --- a/aarch64/src/main.rs +++ b/aarch64/src/main.rs @@ -11,14 +11,18 @@ mod allocator; mod devcons; mod deviceutil; +mod gic; mod io; +mod irq; mod kmem; mod mailbox; mod pagealloc; mod param; mod pre_mmu; +mod reg; mod registers; mod swtch; +mod timer; mod trap; mod uartmini; mod uartpl011; @@ -29,10 +33,12 @@ extern crate alloc; use alloc::boxed::Box; use core::ptr::null_mut; +use core::time::Duration; use param::KZERO; use port::fdt::DeviceTree; use port::mem::{PhysAddr, PhysRange, VirtRange}; -use port::println; +use port::{iprintln, println}; +use timer::{Timer, TimerCallback}; use vm::{Entry, RootPageTableType, VaMapping}; use crate::kmem::{ @@ -103,13 +109,14 @@ fn print_stacks() { let interrupt_stack_max = interrupt_stack_base + unsafe { interruptstacksz.as_ptr().addr() }; let range = VirtRange::new(interrupt_stack_base, interrupt_stack_max); let range_size = range.size(); - println!("Interrupt stack:\t {range} ({range_size:#x})"); + println!("Interrupt stack:{range} ({range_size:#x})"); } /// dtb_va is the virtual address of the DTB structure. The physical address is /// assumed to be dtb_va-KZERO. #[unsafe(no_mangle)] pub extern "C" fn main9(dtb_va: usize) { + irq::init(); trap::init(); // Parse the DTB before we set up memory so we can correctly map it @@ -130,11 +137,11 @@ pub extern "C" fn main9(dtb_va: usize) { devcons::init(&dt); mailbox::init(&dt); + gic::init(&dt); + timer::init(); println!(); println!("r9 from the Internet"); - println!("DTB found at: {:#x}", dtb_va); - println!("midr_el1: {:?}", registers::MidrEl1::read()); print_stacks(); @@ -143,9 +150,6 @@ pub extern "C" fn main9(dtb_va: usize) { print_board_info(); print_memory_info(); - // vmdebug::print_recursive_tables(RootPageTableType::Kernel); - // vmdebug::print_recursive_tables(RootPageTableType::User); - { let page_table = vm::kernel_pagetable(); let entry = Entry::rw_kernel_data(); @@ -167,9 +171,6 @@ pub extern "C" fn main9(dtb_va: usize) { } } - // vmdebug::print_recursive_tables(RootPageTableType::Kernel); - // vmdebug::print_recursive_tables(RootPageTableType::User); - println!("Set up a user process"); unsafe { @@ -177,13 +178,17 @@ pub extern "C" fn main9(dtb_va: usize) { vm::switch(vm::user_pagetable(), RootPageTableType::User); } - test_sysexit(); + // test_sysexit(); - vmdebug::print_recursive_tables(RootPageTableType::Kernel); - vmdebug::print_recursive_tables(RootPageTableType::User); + // vmdebug::print_recursive_tables(RootPageTableType::Kernel); + // vmdebug::print_recursive_tables(RootPageTableType::User); let _b = Box::new("ddododo"); + PC1_TIMER.start(); + PC2_TIMER.start(); + STOP_PC1_TIMER.start(); + println!("looping now"); #[allow(clippy::empty_loop)] @@ -192,6 +197,55 @@ pub extern "C" fn main9(dtb_va: usize) { mod runtime; +// Temp, test-related code + +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Prints ":" each firing; stops itself after `limit` +/// extra firings (0 = run until cancelled). +struct Ticker { + name: &'static str, + counter: AtomicU32, + limit: u32, +} + +impl TimerCallback for Ticker { + fn fire(&self) -> bool { + let n = self.counter.fetch_add(1, Ordering::Relaxed) + 1; + iprintln!("{}:{}", self.name, n); + if self.limit == 0 || n <= self.limit { + true + } else { + iprintln!("stopping {}", self.name); + false + } + } +} + +/// One-shot callback that cancels another timer. +struct CancelTimer { + victim: &'static Timer, + msg: &'static str, +} + +impl TimerCallback for CancelTimer { + fn fire(&self) -> bool { + iprintln!("{}", self.msg); + self.victim.cancel(); + false + } +} + +static PC1: Ticker = Ticker { name: "pc1", counter: AtomicU32::new(0), limit: 0 }; +static PC1_TIMER: Timer = Timer::periodic(Duration::from_secs(1), &PC1); + +static PC2: Ticker = Ticker { name: "pc2", counter: AtomicU32::new(0), limit: 3 }; +static PC2_TIMER: Timer = Timer::periodic(Duration::from_secs(2), &PC2); + +static STOP_PC1: CancelTimer = CancelTimer { victim: &PC1_TIMER, msg: "stopping pc1" }; +static STOP_PC1_TIMER: Timer = Timer::new(Duration::from_secs(5), &STOP_PC1); + +#[allow(dead_code)] fn test_sysexit() { let page_table = vm::user_pagetable(); diff --git a/aarch64/src/reg/cnt_el0.rs b/aarch64/src/reg/cnt_el0.rs new file mode 100644 index 0000000..922ee9b --- /dev/null +++ b/aarch64/src/reg/cnt_el0.rs @@ -0,0 +1,89 @@ +use core::fmt; + +use aarch64_cpu::registers::{ + CNTFRQ_EL0, CNTP_CTL_EL0, CNTP_CVAL_EL0, CNTPCT_EL0, Readable, Writeable, +}; +use bitstruct::bitstruct; + +// CNTPCT_EL0 — 64-bit physical counter +#[derive(Copy, Clone, Default)] +pub struct CntPctEl0(u64); + +impl CntPctEl0 { + pub fn read() -> Self { + Self(if cfg!(test) { 0 } else { CNTPCT_EL0.extract().into() }) + } + + pub fn value(self) -> u64 { + self.0 + } +} + +impl fmt::Debug for CntPctEl0 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("CntPctEl0").field(&format_args!("{}", self.0)).finish() + } +} + +// CNTFRQ_EL0 — timer frequency in Hz +#[derive(Copy, Clone, Default)] +pub struct CntFrqEl0(u64); + +impl CntFrqEl0 { + pub fn read() -> Self { + Self(if cfg!(test) { 0 } else { CNTFRQ_EL0.extract().into() }) + } + + pub fn freq(self) -> u64 { + self.0 + } +} + +impl fmt::Debug for CntFrqEl0 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CntFrqEl0").field("hz", &format_args!("{}", self.0)).finish() + } +} + +// CNTP_CVAL_EL0 — physical timer compare value (write-only for now) +pub struct CntpCvalEl0; + +impl CntpCvalEl0 { + pub fn write(value: u64) { + if !cfg!(test) { + CNTP_CVAL_EL0.set(value); + } + } +} + +// CNTP_CTL_EL0 — physical timer control register +bitstruct! { + #[derive(Copy, Clone, Default)] + pub struct CntpCtlEl0(pub u64) { + pub enable: bool = 0..1; + pub imask: bool = 1..2; + pub istatus: bool = 2..3; + } +} + +impl CntpCtlEl0 { + pub fn read() -> Self { + Self(if cfg!(test) { 0 } else { CNTP_CTL_EL0.extract().into() }) + } + + pub fn write(self) { + if !cfg!(test) { + CNTP_CTL_EL0.set(self.0); + } + } +} + +impl fmt::Debug for CntpCtlEl0 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CntPctl") + .field("enable", &self.enable()) + .field("imask", &self.imask()) + .field("istatus", &self.istatus()) + .finish() + } +} diff --git a/aarch64/src/reg/esr_el1.rs b/aarch64/src/reg/esr_el1.rs new file mode 100644 index 0000000..ff65a31 --- /dev/null +++ b/aarch64/src/reg/esr_el1.rs @@ -0,0 +1,161 @@ +use core::fmt; + +use bitstruct::bitstruct; +use num_enum::TryFromPrimitive; + +bitstruct! { + #[derive(Copy, Clone)] + pub struct EsrEl1(pub u64) { + pub iss: u32 = 0..25; + pub il: bool = 25; + pub ec: u8 = 26..32; + pub iss2: u8 = 32..37; + } +} + +impl EsrEl1 { + /// Try to convert the error into an ExceptionClass enum, or return the original number + /// as the error. + pub fn exception_class_enum(&self) -> Result { + ExceptionClass::try_from(self.ec()).map_err(|e| e.number) + } +} + +impl fmt::Debug for EsrEl1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EsrEl1") + .field("iss", &format_args!("{:#010x}", self.iss())) + .field("il", &format_args!("{}", self.il())) + .field("ec", &format_args!("{:?}", self.exception_class_enum())) + .field("iss2", &format_args!("{:#04x}", self.iss2())) + .finish() + } +} + +/// Exception class maps to ESR_EL1 EC bits[31:26]. We skip aarch32 exceptions. +#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] +#[repr(u8)] +pub enum ExceptionClass { + Unknown = 0, + WaitFor = 1, + FloatSimd = 7, + Ls64 = 10, + BranchTargetException = 13, + IllegalExecutionState = 14, + MsrMrsSystem = 24, + Sve = 25, + Tstart = 27, + PointerAuthFailure = 28, + Sme = 29, + GranuleProtectionCheck = 30, + InstructionAbortLowerEl = 32, + InstructionAbortSameEl = 33, + PcAlignmentFault = 34, + DataAbortLowerEl = 36, + DataAbortSameEl = 37, + SpAlignmentFault = 38, + MemoryOperationException = 39, + TrappedFloatingPointException = 44, + SError = 47, + BreakpointLowerEl = 48, + BreakpointSameEl = 49, + SoftwareStepLowerEl = 50, + SoftwareStepSameEl = 51, + WatchpointLowerEl = 52, + WatchpointSameEl = 53, + Brk = 60, +} + +bitstruct! { + #[derive(Copy, Clone)] + pub struct EsrEl1IssInstructionAbort(pub u32) { + ifsc: u8 = 0..6; + s1ptw: bool = 7; + ea: bool = 9; + fnv: bool = 10; + set: u8 = 11..13; + } +} + +#[allow(dead_code)] +impl EsrEl1IssInstructionAbort { + pub fn from_esr_el1(r: EsrEl1) -> Option { + r.exception_class_enum() + .ok() + .filter(|ec| *ec == ExceptionClass::InstructionAbortSameEl) + .map(|_| EsrEl1IssInstructionAbort(r.iss())) + } + + pub fn instruction_fault(&self) -> Result { + InstructionFaultStatusCode::try_from(self.ifsc()).map_err(|e| e.number) + } +} + +#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] +#[repr(u8)] +pub enum InstructionFaultStatusCode { + AddressSizeFaultLevel0 = 0, + AddressSizeFaultLevel1 = 1, + AddressSizeFaultLevel2 = 2, + AddressSizeFaultLevel3 = 3, + TranslationFaultLevel0 = 4, + TranslationFaultLevel1 = 5, + TranslationFaultLevel2 = 6, + TranslationFaultLevel3 = 7, + AccessFlagFaultLevel0 = 8, + AccessFlagFaultLevel1 = 9, + AccessFlagFaultLevel2 = 10, + AccessFlagFaultLevel3 = 11, + PermissionFaultLevel0 = 12, + PermissionFaultLevel1 = 13, + PermissionFaultLevel2 = 14, + PermissionFaultLevel3 = 15, + SyncExtAbortNotOnWalkOrUpdate = 16, + SyncExtAbortOnWalkOrUpdateLevelNeg1 = 19, + SyncExtAbortOnWalkOrUpdateLevel0 = 20, + SyncExtAbortOnWalkOrUpdateLevel1 = 21, + SyncExtAbortOnWalkOrUpdateLevel2 = 22, + SyncExtAbortOnWalkOrUpdateLevel3 = 23, + SyncParityOrEccErrOnMemAccessNotOnWalk = 24, + SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevelNeg1 = 27, + SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel0 = 28, + SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel1 = 29, + SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel2 = 30, + SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel3 = 31, + GranuleProtectFaultOnWalkOrUpdateLevelNeg1 = 35, + GranuleProtectFaultOnWalkOrUpdateLevel0 = 36, + GranuleProtectFaultOnWalkOrUpdateLevel1 = 37, + GranuleProtectFaultOnWalkOrUpdateLevel2 = 38, + GranuleProtectFaultOnWalkOrUpdateLevel3 = 39, + GranuleProtectFaultNotOnWalkOrUpdateLevel = 40, + AddressSizeFaultLevelNeg1 = 41, + TranslationFaultLevelNeg1 = 43, + TlbConflictAbort = 48, + UnsupportedAtomicHardwareUpdateFault = 49, +} + +#[cfg(test)] +mod tests { + use super::*; + + // This test is useful for making sense of early-stage exceptions. Qemu + // will report an exception of the form below. Copy the ESR value into + // this test to break it down. + // + // Exception return from AArch64 EL2 to AArch64 EL1 PC 0x8006c + // Taking exception 3 [Prefetch Abort] on CPU 0 + // ...from EL1 to EL1 + // ...with ESR 0x21/0x86000004 + // ...with FAR 0x80090 + // ...with ELR 0x80090 + // ...to EL1 PC 0x200 PSTATE 0x3c5 + #[test] + fn test_parse_esr_el1() { + let r = EsrEl1(0x86000004); + assert_eq!(r.exception_class_enum().unwrap(), ExceptionClass::InstructionAbortSameEl); + assert_eq!( + EsrEl1IssInstructionAbort::from_esr_el1(r).unwrap().instruction_fault().unwrap(), + InstructionFaultStatusCode::TranslationFaultLevel0 + ); + } +} diff --git a/aarch64/src/reg/midr_el1.rs b/aarch64/src/reg/midr_el1.rs new file mode 100644 index 0000000..9d23e13 --- /dev/null +++ b/aarch64/src/reg/midr_el1.rs @@ -0,0 +1,53 @@ +use core::fmt; + +use aarch64_cpu::registers::{MIDR_EL1, Readable}; +use bitstruct::bitstruct; +use num_enum::TryFromPrimitive; + +bitstruct! { + #[derive(Copy, Clone)] + pub struct MidrEl1(pub u64) { + revision: u8 = 0..4; + partnum: u16 = 4..16; + architecture: u8 = 16..20; + variant: u8 = 20..24; + implementer: u16 = 24..32; + } +} + +#[allow(dead_code)] +impl MidrEl1 { + pub fn read() -> Self { + Self(if cfg!(test) { 0 } else { MIDR_EL1.extract().into() }) + } + + pub fn partnum_enum(&self) -> Result { + PartNum::try_from(self.partnum()).map_err(|e| e.number) + } +} + +impl fmt::Debug for MidrEl1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MidrEl1") + .field("revision", &format_args!("{:#x}", self.revision())) + .field( + "partnum", + &format_args!("{:?}", self.partnum_enum().unwrap_or(PartNum::Unknown)), + ) + .field("architecture", &format_args!("{:#x}", self.architecture())) + .field("variant", &format_args!("{:#x}", self.variant())) + .field("implementer", &format_args!("{:#x}", self.implementer())) + .finish() + } +} + +/// Known IDs for midr_el1's partnum +#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] +#[repr(u16)] +pub enum PartNum { + Unknown = 0, + RaspberryPi1 = 0xb76, + RaspberryPi2 = 0xc07, + RaspberryPi3 = 0xd03, + RaspberryPi4 = 0xd08, +} diff --git a/aarch64/src/reg/mod.rs b/aarch64/src/reg/mod.rs new file mode 100644 index 0000000..520c023 --- /dev/null +++ b/aarch64/src/reg/mod.rs @@ -0,0 +1,4 @@ +#[allow(dead_code)] +pub mod cnt_el0; +pub mod esr_el1; +pub mod midr_el1; diff --git a/aarch64/src/registers.rs b/aarch64/src/registers.rs index cb00320..e166f95 100644 --- a/aarch64/src/registers.rs +++ b/aarch64/src/registers.rs @@ -1,241 +1,6 @@ #![allow(non_upper_case_globals)] -use aarch64_cpu::registers::MIDR_EL1; -use aarch64_cpu::registers::Readable; -use bitstruct::bitstruct; -use core::fmt; -use num_enum::TryFromPrimitive; - // GPIO registers pub const GPFSEL1: usize = 0x04; // GPIO function select register 1 pub const GPPUD: usize = 0x94; // GPIO pin pull up/down enable pub const GPPUDCLK0: usize = 0x98; // GPIO pin pull up/down enable clock 0 - -// UART 0 (PL011) registers -#[allow(unused)] -pub const UART0_DR: usize = 0x00; // Data register -#[allow(unused)] -pub const UART0_FR: usize = 0x18; // Flag register -pub const UART0_IBRD: usize = 0x24; // Integer baud rate divisor -pub const UART0_FBRD: usize = 0x28; // Fractional baud rate divisor -pub const UART0_LCRH: usize = 0x2c; // Line control register -pub const UART0_CR: usize = 0x30; // Control register -pub const UART0_IMSC: usize = 0x38; // Interrupt mask set clear register -pub const UART0_ICR: usize = 0x44; // Interrupt clear register - -// AUX registers, offset from aux_reg -pub const AUX_ENABLE: usize = 0x04; // AUX enable register (Mini Uart, SPIs) - -// UART1 registers, offset from miniuart_reg -pub const AUX_MU_IO: usize = 0x00; // AUX IO data register -pub const AUX_MU_IER: usize = 0x04; // Mini Uart interrupt enable register -pub const AUX_MU_IIR: usize = 0x08; // Mini Uart interrupt identify register -pub const AUX_MU_LCR: usize = 0x0c; // Mini Uart line control register -pub const AUX_MU_MCR: usize = 0x10; // Mini Uart line control register -pub const AUX_MU_LSR: usize = 0x14; // Mini Uart line status register -pub const AUX_MU_CNTL: usize = 0x20; // Mini Uart control register -pub const AUX_MU_BAUD: usize = 0x28; // Mini Uart baudrate register - -bitstruct! { - #[derive(Copy, Clone)] - pub struct MidrEl1(pub u64) { - revision: u8 = 0..4; - partnum: u16 = 4..16; - architecture: u8 = 16..20; - variant: u8 = 20..24; - implementer: u16 = 24..32; - } -} - -impl MidrEl1 { - pub fn read() -> Self { - Self(if cfg!(test) { 0 } else { MIDR_EL1.extract().into() }) - } - - pub fn partnum_enum(&self) -> Result { - PartNum::try_from(self.partnum()).map_err(|e| e.number) - } -} - -impl fmt::Debug for MidrEl1 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("MidrEl1") - .field("revision", &format_args!("{:#x}", self.revision())) - .field( - "partnum", - &format_args!("{:?}", self.partnum_enum().unwrap_or(PartNum::Unknown)), - ) - .field("architecture", &format_args!("{:#x}", self.architecture())) - .field("variant", &format_args!("{:#x}", self.variant())) - .field("implementer", &format_args!("{:#x}", self.implementer())) - .finish() - } -} - -/// Known IDs for midr_el1's partnum -#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] -#[repr(u16)] -pub enum PartNum { - Unknown = 0, - RaspberryPi1 = 0xb76, - RaspberryPi2 = 0xc07, - RaspberryPi3 = 0xd03, - RaspberryPi4 = 0xd08, -} - -bitstruct! { - #[derive(Copy, Clone)] - pub struct EsrEl1(pub u64) { - pub iss: u32 = 0..25; - pub il: bool = 25; - pub ec: u8 = 26..32; - pub iss2: u8 = 32..37; - } -} - -impl EsrEl1 { - /// Try to convert the error into an ExceptionClass enum, or return the original number - /// as the error. - pub fn exception_class_enum(&self) -> Result { - ExceptionClass::try_from(self.ec()).map_err(|e| e.number) - } -} - -impl fmt::Debug for EsrEl1 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("EsrEl1") - .field("iss", &format_args!("{:#010x}", self.iss())) - .field("il", &format_args!("{}", self.il())) - .field("ec", &format_args!("{:?}", self.exception_class_enum())) - .field("iss2", &format_args!("{:#04x}", self.iss2())) - .finish() - } -} - -/// Exception class maps to ESR_EL1 EC bits[31:26]. We skip aarch32 exceptions. -#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] -#[repr(u8)] -pub enum ExceptionClass { - Unknown = 0, - WaitFor = 1, - FloatSimd = 7, - Ls64 = 10, - BranchTargetException = 13, - IllegalExecutionState = 14, - MsrMrsSystem = 24, - Sve = 25, - Tstart = 27, - PointerAuthFailure = 28, - Sme = 29, - GranuleProtectionCheck = 30, - InstructionAbortLowerEl = 32, - InstructionAbortSameEl = 33, - PcAlignmentFault = 34, - DataAbortLowerEl = 36, - DataAbortSameEl = 37, - SpAlignmentFault = 38, - MemoryOperationException = 39, - TrappedFloatingPointException = 44, - SError = 47, - BreakpointLowerEl = 48, - BreakpointSameEl = 49, - SoftwareStepLowerEl = 50, - SoftwareStepSameEl = 51, - WatchpointLowerEl = 52, - WatchpointSameEl = 53, - Brk = 60, -} - -bitstruct! { - #[derive(Copy, Clone)] - pub struct EsrEl1IssInstructionAbort(pub u32) { - ifsc: u8 = 0..6; - s1ptw: bool = 7; - ea: bool = 9; - fnv: bool = 10; - set: u8 = 11..13; - } -} - -#[allow(dead_code)] -impl EsrEl1IssInstructionAbort { - pub fn from_esr_el1(r: EsrEl1) -> Option { - r.exception_class_enum() - .ok() - .filter(|ec| *ec == ExceptionClass::InstructionAbortSameEl) - .map(|_| EsrEl1IssInstructionAbort(r.iss())) - } - - pub fn instruction_fault(&self) -> Result { - InstructionFaultStatusCode::try_from(self.ifsc()).map_err(|e| e.number) - } -} - -#[derive(Debug, Eq, PartialEq, TryFromPrimitive)] -#[repr(u8)] -pub enum InstructionFaultStatusCode { - AddressSizeFaultLevel0 = 0, - AddressSizeFaultLevel1 = 1, - AddressSizeFaultLevel2 = 2, - AddressSizeFaultLevel3 = 3, - TranslationFaultLevel0 = 4, - TranslationFaultLevel1 = 5, - TranslationFaultLevel2 = 6, - TranslationFaultLevel3 = 7, - AccessFlagFaultLevel0 = 8, - AccessFlagFaultLevel1 = 9, - AccessFlagFaultLevel2 = 10, - AccessFlagFaultLevel3 = 11, - PermissionFaultLevel0 = 12, - PermissionFaultLevel1 = 13, - PermissionFaultLevel2 = 14, - PermissionFaultLevel3 = 15, - SyncExtAbortNotOnWalkOrUpdate = 16, - SyncExtAbortOnWalkOrUpdateLevelNeg1 = 19, - SyncExtAbortOnWalkOrUpdateLevel0 = 20, - SyncExtAbortOnWalkOrUpdateLevel1 = 21, - SyncExtAbortOnWalkOrUpdateLevel2 = 22, - SyncExtAbortOnWalkOrUpdateLevel3 = 23, - SyncParityOrEccErrOnMemAccessNotOnWalk = 24, - SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevelNeg1 = 27, - SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel0 = 28, - SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel1 = 29, - SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel2 = 30, - SyncParityOrEccErrOnMemAccessOnWalkOrUpdateLevel3 = 31, - GranuleProtectFaultOnWalkOrUpdateLevelNeg1 = 35, - GranuleProtectFaultOnWalkOrUpdateLevel0 = 36, - GranuleProtectFaultOnWalkOrUpdateLevel1 = 37, - GranuleProtectFaultOnWalkOrUpdateLevel2 = 38, - GranuleProtectFaultOnWalkOrUpdateLevel3 = 39, - GranuleProtectFaultNotOnWalkOrUpdateLevel = 40, - AddressSizeFaultLevelNeg1 = 41, - TranslationFaultLevelNeg1 = 43, - TlbConflictAbort = 48, - UnsupportedAtomicHardwareUpdateFault = 49, -} - -#[cfg(test)] -mod tests { - use super::*; - - // This test is useful for making sense of early-stage exceptions. Qemu - // will report an exception of the form below. Copy the ESR value into - // this test to break it down. - // - // Exception return from AArch64 EL2 to AArch64 EL1 PC 0x8006c - // Taking exception 3 [Prefetch Abort] on CPU 0 - // ...from EL1 to EL1 - // ...with ESR 0x21/0x86000004 - // ...with FAR 0x80090 - // ...with ELR 0x80090 - // ...to EL1 PC 0x200 PSTATE 0x3c5 - #[test] - fn test_parse_esr_el1() { - let r = EsrEl1(0x86000004); - assert_eq!(r.exception_class_enum().unwrap(), ExceptionClass::InstructionAbortSameEl); - assert_eq!( - EsrEl1IssInstructionAbort::from_esr_el1(r).unwrap().instruction_fault().unwrap(), - InstructionFaultStatusCode::TranslationFaultLevel0 - ); - } -} diff --git a/aarch64/src/runtime.rs b/aarch64/src/runtime.rs index b9320c9..f13ec75 100644 --- a/aarch64/src/runtime.rs +++ b/aarch64/src/runtime.rs @@ -6,13 +6,13 @@ use alloc::alloc::Layout; use core::panic::PanicInfo; #[cfg(not(test))] -use port::println; +use port::iprintln; // TODO // - Add qemu integration test #[panic_handler] pub fn panic(info: &PanicInfo) -> ! { - println!("{}\n", info); + iprintln!("{}\n", info); #[allow(clippy::empty_loop)] loop {} diff --git a/aarch64/src/timer.rs b/aarch64/src/timer.rs new file mode 100644 index 0000000..428f103 --- /dev/null +++ b/aarch64/src/timer.rs @@ -0,0 +1,200 @@ +//! Minimal timer subsystem using the ARMv8 architectural timer. +//! +//! Timers are caller-owned: the subsystem stores only `&'static` +//! references in a small fixed table, never allocates, and the +//! interrupt handler frees nothing. Timers therefore live in statics, +//! with interior mutability making them shareable with the handler. + +use core::ptr; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; + +use port::irq::IrqGuard; +use port::mcslock::{Lock, LockNode}; + +use crate::reg::cnt_el0::{CntFrqEl0, CntPctEl0, CntpCtlEl0, CntpCvalEl0}; + +/// Fired in interrupt context. Return true to keep a periodic timer +/// running; the return value is ignored for one-shot timers. +pub trait TimerCallback: Send + Sync { + fn fire(&self) -> bool; +} + +/// A caller-owned timer. `start` registers it with the subsystem, +/// which only ever borrows it. Not designed for concurrent restarts +/// of the same timer. +pub struct Timer { + duration: Duration, + repeat: bool, + deadline_ticks: AtomicU64, + period_ticks: AtomicU64, + active: AtomicBool, + callback: &'static dyn TimerCallback, +} + +impl Timer { + /// A one-shot timer firing once, `relative` after `start`. + pub const fn new(relative: Duration, callback: &'static dyn TimerCallback) -> Self { + Self { + duration: relative, + repeat: false, + deadline_ticks: AtomicU64::new(0), + period_ticks: AtomicU64::new(0), + active: AtomicBool::new(false), + callback, + } + } + + /// A timer firing every `period` until its callback returns false + /// or it is cancelled. + pub const fn periodic(period: Duration, callback: &'static dyn TimerCallback) -> Self { + let mut timer = Self::new(period, callback); + timer.repeat = true; + timer + } + + /// Register and arm the timer. Panics if the timer table is full. + pub fn start(&'static self) { + let ticks = duration_to_ticks(self.duration); + self.period_ticks.store(if self.repeat { ticks } else { 0 }, Ordering::Relaxed); + self.deadline_ticks.store(now() + ticks, Ordering::Relaxed); + self.active.store(true, Ordering::Release); + let _irq = IrqGuard::new(); + register(self); + arm_hardware(); + } + + /// Deactivate the timer. Lazy: an already-armed hardware deadline + /// may still cause one spurious wakeup. Idempotent, and safe to + /// call from any callback, including the timer's own. + pub fn cancel(&self) { + self.active.store(false, Ordering::Release); + } +} + +fn timer_enable() { + if !cfg!(test) { + CntpCtlEl0::write(CntpCtlEl0::read().with_enable(true)); + } +} + +fn timer_disable() { + if !cfg!(test) { + CntpCtlEl0::write(CntpCtlEl0::read().with_enable(false)); + } +} + +const MAX_TIMERS: usize = 8; + +static TIMERS: Lock<[Option<&'static Timer>; MAX_TIMERS]> = Lock::new("timers", [None; MAX_TIMERS]); + +/// Run `f` with the timer table locked and IRQs masked. The lock is +/// shared with the interrupt handler, so it must never be held with +/// IRQs enabled: a timer interrupt arriving mid-hold would spin on it +/// forever. (In the handler itself the masking is a harmless no-op.) +fn with_timers(f: impl FnOnce(&mut [Option<&'static Timer>; MAX_TIMERS]) -> R) -> R { + let _irq = IrqGuard::new(); + let node = LockNode::new(); + let mut guard = TIMERS.lock(&node); + f(&mut guard) +} + +fn register(timer: &'static Timer) { + with_timers(|timers| { + // Already registered (a restart)? + if timers.iter().flatten().any(|t| ptr::eq(*t, timer)) { + return; + } + // Take a free slot, or one whose timer is no longer active. + for slot in timers.iter_mut() { + if slot.is_none_or(|t| !t.active.load(Ordering::Acquire)) { + *slot = Some(timer); + return; + } + } + panic!("timer table full"); + }) +} + +/// Arm the hardware for the earliest active deadline, or disarm if +/// there is none. Call with IRQs masked. +fn arm_hardware() { + let timers = with_timers(|timers| *timers); + let next = timers + .into_iter() + .flatten() + .filter(|t| t.active.load(Ordering::Acquire)) + .map(|t| t.deadline_ticks.load(Ordering::Relaxed)) + .min(); + match next { + Some(deadline) => { + CntpCvalEl0::write(deadline); + timer_enable(); + } + None => timer_disable(), + } +} + +static TIMER_FREQ: AtomicU64 = AtomicU64::new(0); + +pub fn init() { + let freq = CntFrqEl0::read().freq(); + if freq == 0 { + panic!("timer: CNTFRQ_EL0=0: counter frequency not programmed by firmware"); + } + TIMER_FREQ.store(freq, Ordering::Relaxed); +} + +fn now() -> u64 { + CntPctEl0::read().value() +} + +/// Convert a duration to hardware ticks. A zero tick count would arm +/// a periodic timer with an unchanging, always-past deadline — an +/// interrupt storm — so a timer started before `init` is a bug. +fn duration_to_ticks(dur: Duration) -> u64 { + let freq = TIMER_FREQ.load(Ordering::Relaxed); + ((dur.as_nanos() * freq as u128) / 1_000_000_000) as u64 +} + +/// Hardware interrupt handler — called from the trap handler. +/// +/// Fires all due timers (outside the table lock, so a callback may +/// start or cancel timers) and arms the next deadline, which is also +/// what deasserts the level-triggered timer interrupt: the new CVAL is +/// in the future, or the timer is disabled. +pub fn interrupt_handler() { + if cfg!(test) { + return; + } + + // 1. Copy out the table so callbacks run outside the lock. + let timers = with_timers(|timers| *timers); + + // 2. Fire due timers. + let now = now(); + for timer in timers.into_iter().flatten() { + if !timer.active.load(Ordering::Acquire) { + continue; + } + let deadline = timer.deadline_ticks.load(Ordering::Relaxed); + if deadline > now { + continue; + } + let period = timer.period_ticks.load(Ordering::Relaxed); + if period == 0 { + // Deactivate before firing so the callback may restart it. + timer.active.store(false, Ordering::Release); + timer.callback.fire(); + } else if timer.callback.fire() { + // Advance the deadline; if the callback cancelled its own + // timer the cleared active flag still stops it. + timer.deadline_ticks.store(deadline + period, Ordering::Relaxed); + } else { + timer.active.store(false, Ordering::Release); + } + } + + // 3. Arm next timer or disarm, deasserting the interrupt. + arm_hardware(); +} diff --git a/aarch64/src/trap.rs b/aarch64/src/trap.rs index c1aa41b..49ede65 100644 --- a/aarch64/src/trap.rs +++ b/aarch64/src/trap.rs @@ -1,7 +1,8 @@ use core::fmt; -use crate::registers::EsrEl1; -use port::println; +use crate::reg::esr_el1::EsrEl1; +use crate::{gic, timer}; +use port::iprintln; #[cfg(not(test))] core::arch::global_asm!(include_str!("trap.S")); @@ -103,17 +104,32 @@ impl fmt::Debug for TrapFrame { #[unsafe(no_mangle)] pub extern "C" fn trap_unsafe(frame: *mut TrapFrame) { + port::irq::enter_interrupt(); unsafe { trap(frame.as_mut().unwrap()) } + port::irq::exit_interrupt(); } fn trap(frame: &mut TrapFrame) { + if let Some(iar) = gic::try_ack_interrupt() { + match iar.int_id() { + gic::TIMER_INTID => timer::interrupt_handler(), + intid => { + iprintln!("Unhandled GIC IRQ {intid}"); + // Disable to avoid repeated unhandled interrupts + gic::disable_interrupt(intid); + } + } + gic::end_interrupt(iar); + return; + } + if frame.esr_el1.ec() == 0x15 { // Syscall let syscallid = frame.esr_el1.iss(); - println!("Syscall {syscallid}"); + iprintln!("Syscall {syscallid}"); } else { - println!("{:#?}", frame); - println!("Unhandled interrupt"); + iprintln!("{:#?}", frame); + iprintln!("Unhandled interrupt"); } loop { diff --git a/aarch64/src/uartmini.rs b/aarch64/src/uartmini.rs index fa5ab8a..909c674 100644 --- a/aarch64/src/uartmini.rs +++ b/aarch64/src/uartmini.rs @@ -5,15 +5,25 @@ use port::mem::{PhysRange, VirtRange}; use crate::deviceutil::map_device_register; use crate::io::{delay, read_reg, write_or_reg, write_reg}; -use crate::registers::{ - AUX_ENABLE, AUX_MU_BAUD, AUX_MU_CNTL, AUX_MU_IER, AUX_MU_IIR, AUX_MU_IO, AUX_MU_LCR, - AUX_MU_LSR, AUX_MU_MCR, GPFSEL1, GPPUD, GPPUDCLK0, -}; +use crate::registers::{GPFSEL1, GPPUD, GPPUDCLK0}; use crate::vm; #[cfg(not(test))] use port::println; +// AUX registers, offset from aux_reg +pub const AUX_ENABLE: usize = 0x04; // AUX enable register (Mini Uart, SPIs) + +// UART1 registers, offset from miniuart_reg +pub const AUX_MU_IO: usize = 0x00; // AUX IO data register +pub const AUX_MU_IER: usize = 0x04; // Mini Uart interrupt enable register +pub const AUX_MU_IIR: usize = 0x08; // Mini Uart interrupt identify register +pub const AUX_MU_LCR: usize = 0x0c; // Mini Uart line control register +pub const AUX_MU_MCR: usize = 0x10; // Mini Uart line control register +pub const AUX_MU_LSR: usize = 0x14; // Mini Uart line status register +pub const AUX_MU_CNTL: usize = 0x20; // Mini Uart control register +pub const AUX_MU_BAUD: usize = 0x28; // Mini Uart baudrate register + /// MiniUart is assigned to UART1 on the Raspberry Pi. It is easier to use with /// real hardware, as it requires no additional configuration. Conversely, it's /// harded to use with QEMU, as it can't be used with the `nographic` switch. diff --git a/aarch64/src/uartpl011.rs b/aarch64/src/uartpl011.rs index 9157cce..77ad6c0 100644 --- a/aarch64/src/uartpl011.rs +++ b/aarch64/src/uartpl011.rs @@ -1,9 +1,6 @@ use crate::deviceutil::map_device_register; use crate::io::{GpioPull, delay, read_reg, write_reg}; -use crate::registers::{ - GPPUD, GPPUDCLK0, UART0_CR, UART0_DR, UART0_FBRD, UART0_FR, UART0_IBRD, UART0_ICR, UART0_IMSC, - UART0_LCRH, -}; +use crate::registers::{GPPUD, GPPUDCLK0}; use crate::{mailbox, vm}; use port::Result; use port::devcons::Uart; @@ -13,6 +10,18 @@ use port::mem::{PhysRange, VirtRange}; #[cfg(not(test))] use port::println; +// UART 0 (PL011) registers +#[allow(unused)] +pub const UART0_DR: usize = 0x00; // Data register +#[allow(unused)] +pub const UART0_FR: usize = 0x18; // Flag register +pub const UART0_IBRD: usize = 0x24; // Integer baud rate divisor +pub const UART0_FBRD: usize = 0x28; // Fractional baud rate divisor +pub const UART0_LCRH: usize = 0x2c; // Line control register +pub const UART0_CR: usize = 0x30; // Control register +pub const UART0_IMSC: usize = 0x38; // Interrupt mask set clear register +pub const UART0_ICR: usize = 0x44; // Interrupt clear register + #[allow(dead_code)] pub struct Pl011Uart { gpio_virtrange: VirtRange, diff --git a/aarch64/src/vm.rs b/aarch64/src/vm.rs index 7372fd4..8b9529a 100644 --- a/aarch64/src/vm.rs +++ b/aarch64/src/vm.rs @@ -516,7 +516,7 @@ impl RootPageTable { static next_free_device_page_va: AtomicUsize = AtomicUsize::new(KZERO + 0x100000000000); pub fn next_free_device_page4k() -> VaMapping { next_free_device_page_va - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { Some(current + PageSize::Page4K.size()) }) .map(VaMapping::Addr) diff --git a/aarch64/src/vmdebug.rs b/aarch64/src/vmdebug.rs index 96d0e08..484b95a 100644 --- a/aarch64/src/vmdebug.rs +++ b/aarch64/src/vmdebug.rs @@ -88,6 +88,7 @@ fn recursive_root_page_table(pgtype: RootPageTableType) -> &'static mut RootPage } /// Recursively write out all the tables and all its children +#[allow(dead_code)] pub fn print_recursive_tables(pgtype: RootPageTableType) { let root_page_table = recursive_root_page_table(pgtype); println!("Root va:{:018p}", root_page_table); diff --git a/lib/x86_64-unknown-none-elf.json b/lib/x86_64-unknown-none-elf.json index 577880f..b1b2ca6 100644 --- a/lib/x86_64-unknown-none-elf.json +++ b/lib/x86_64-unknown-none-elf.json @@ -1,22 +1,22 @@ { - "llvm-target": "x86_64-unknown-none-elf", - "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", - "linker-flavor": "ld.lld", - "linker": "rust-lld", - "target-endian": "little", - "target-pointer-width": 64, - "target-c-int-width": 32, - "arch": "x86_64", - "os": "none", - "executables": true, - "relocation-model": "static", - "code-model": "kernel", - "disable-redzone": true, - "features": "-mmx,-sse,+soft-float", - "panic-strategy": "abort", - "frame-pointer": "always", - "pre-link-args": { - "ld.lld": ["-nostdlib"] - }, - "rustc-abi": "x86-softfloat" + "llvm-target": "x86_64-unknown-none-elf", + "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", + "linker-flavor": "ld.lld", + "linker": "rust-lld", + "target-endian": "little", + "target-pointer-width": 64, + "target-c-int-width": 32, + "arch": "x86_64", + "os": "none", + "executables": true, + "relocation-model": "static", + "code-model": "kernel", + "disable-redzone": true, + "features": "-mmx,-sse,+soft-float", + "panic-strategy": "abort", + "frame-pointer": "always", + "pre-link-args": { + "ld.lld": ["-nostdlib"] + }, + "rustc-abi": "softfloat" } diff --git a/port/src/allocator.rs b/port/src/allocator.rs index f020913..610caa8 100644 --- a/port/src/allocator.rs +++ b/port/src/allocator.rs @@ -88,7 +88,7 @@ impl BumpAlloc { let mut first = ptr::null_mut(); let mut adjust = 0; self.cursor - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| { first = base.wrapping_add(current); adjust = first.align_offset(align); let offset = current.checked_add(adjust).expect("alignment overflow"); @@ -510,6 +510,12 @@ pub mod global { where F: FnOnce(&mut QuickFit) -> R, { + // Interrupt context must not allocate: an interrupt-context + // allocation landing while another allocation has the + // QuickFit pointer checked out would find it null and panic + // only when the timing is unlucky. Catch it deterministically + // here instead. + assert!(!crate::irq::in_interrupt(), "allocation in interrupt context"); let a = self.0.swap(ptr::null_mut(), Ordering::Relaxed); assert!(!a.is_null(), "global allocator is nil"); let r = thunk(unsafe { &mut *a }); diff --git a/port/src/devcons.rs b/port/src/devcons.rs index 8ece44a..a9755f5 100644 --- a/port/src/devcons.rs +++ b/port/src/devcons.rs @@ -1,6 +1,9 @@ use crate::Result; +use crate::irq::IrqGuard; use crate::mcslock::{Lock, LockNode}; use core::fmt; +use core::ptr; +use core::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; const fn ctrl(b: u8) -> u8 { b - b'@' @@ -21,7 +24,7 @@ pub trait Uart { fn putb(&self, b: u8); } -static CONS: Lock> = Lock::new("cons", None); +static CONS: Lock> = Lock::new("cons", None); /// Console is what should be used in almost all cases, as it ensures threadsafe /// use of the console. @@ -30,7 +33,7 @@ pub struct Console; impl Console { pub fn set_uart(uart_fn: F) where - F: FnOnce() -> Result<&'static mut dyn Uart>, + F: FnOnce() -> Result<&'static dyn Uart>, { let node = LockNode::new(); let mut cons = CONS.lock(&node); @@ -40,9 +43,12 @@ impl Console { pub fn putstr(&mut self, s: &str) { // XXX: Just for testing. + // The console lock is thread-context only; interrupt context + // must use iprint, which bypasses it. + debug_assert!(!crate::irq::in_interrupt(), "println in interrupt context; use iprintln"); let node = LockNode::new(); - let mut uart_guard = CONS.lock(&node); - if let Some(uart) = uart_guard.as_deref_mut() { + let uart_guard = CONS.lock(&node); + if let Some(uart) = *uart_guard { for b in s.bytes() { putb(uart, b); } @@ -77,7 +83,7 @@ macro_rules! print { }}; } -fn putb(uart: &mut dyn Uart, b: u8) { +fn putb(uart: &dyn Uart, b: u8) { if b == b'\n' { uart.putb(b'\r'); } else if b == BACKSPACE { @@ -86,3 +92,89 @@ fn putb(uart: &mut dyn Uart, b: u8) { } uart.putb(b); } + +// iprint: the interrupt- and panic-safe print, in the tradition of +// Plan 9's iprint. It masks IRQs, takes only a best-effort interlock, +// and writes polled bytes directly to the hardware, bypassing the +// console lock — so it works in interrupt context, in panic, and while +// debugging the console or locks themselves. Output may interleave +// with a concurrent print; that is the accepted price of never +// blocking. + +/// Direct console byte writer, registered by arch code at boot (same +/// pattern as `irq::set_ops`). Must be polled and lock-free: it is +/// called with no locks held from any context. +pub struct IprintOps { + pub putb: fn(u8), +} + +static IPRINT_OPS: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Register the direct console writer. Call at boot once the console +/// hardware is initialised; until then iprint output is dropped. +pub fn set_iprint_ops(ops: &'static IprintOps) { + IPRINT_OPS.store(ops as *const IprintOps as *mut IprintOps, Ordering::Release); +} + +fn iprint_ops() -> Option<&'static IprintOps> { + unsafe { IPRINT_OPS.load(Ordering::Acquire).as_ref() } +} + +/// Best-effort interlock so concurrent iprints don't interleave. +/// Never required for correctness — see `iprint_trylock`. +static IPRINT_LOCK: AtomicBool = AtomicBool::new(false); + +/// Try to take the interlock, giving up after a bounded spin: if +/// another core holds it too long, print anyway — interleaved output +/// beats a silent core. A same-core holder is impossible, as the lock +/// is only ever held with IRQs masked. +fn iprint_trylock() -> bool { + for _ in 0..1_000_000 { + if IPRINT_LOCK.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed).is_ok() { + return true; + } + core::hint::spin_loop(); + } + false +} + +struct IprintWriter { + putb: fn(u8), +} + +impl fmt::Write for IprintWriter { + fn write_str(&mut self, s: &str) -> fmt::Result { + for b in s.bytes() { + if b == b'\n' { + (self.putb)(b'\r'); + } + (self.putb)(b); + } + Ok(()) + } +} + +pub fn iprint(args: fmt::Arguments) { + let _irq = IrqGuard::new(); + let Some(ops) = iprint_ops() else { + return; + }; + let locked = iprint_trylock(); + let _ = fmt::Write::write_fmt(&mut IprintWriter { putb: ops.putb }, args); + if locked { + IPRINT_LOCK.store(false, Ordering::Release); + } +} + +#[macro_export] +macro_rules! iprintln { + () => ($crate::iprint!("\n")); + ($($arg:tt)*) => ($crate::iprint!("{}\n", format_args!($($arg)*))); +} + +#[macro_export] +macro_rules! iprint { + ($($args:tt)*) => {{ + $crate::devcons::iprint(format_args!($($args)*)) + }}; +} diff --git a/port/src/irq.rs b/port/src/irq.rs new file mode 100644 index 0000000..35eedd6 --- /dev/null +++ b/port/src/irq.rs @@ -0,0 +1,93 @@ +//! Core-local interrupt masking and interrupt-context tracking. +//! +//! Any lock that is also taken in interrupt context (e.g. the console +//! lock) must be held with interrupts masked; otherwise an interrupt +//! arriving while the lock is held leaves the handler spinning on a +//! lock its own core can never release. `IrqGuard` provides that +//! masking as an RAII guard. +//! +//! `in_interrupt` supports the complementary approach for subsystems +//! that interrupt context is simply forbidden to use (e.g. the +//! allocator): assert the invariant instead of masking around it. +//! +//! Masking is architecture-specific, so each arch registers its +//! implementation at early boot via `set_ops`, before enabling +//! interrupts (the pattern devcons uses for the Uart). Until then, and +//! in hosted test builds where the mask instructions would be +//! privileged, `IrqGuard` is a no-op. + +use core::marker::PhantomData; +use core::ptr; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +// Depth rather than a flag so nested exceptions stay counted. +// Core-local in spirit; needs to become per-core state under SMP. +static INTERRUPT_DEPTH: AtomicUsize = AtomicUsize::new(0); + +/// Mark entry to interrupt context. Called by the arch trap handler. +pub fn enter_interrupt() { + INTERRUPT_DEPTH.fetch_add(1, Ordering::Relaxed); +} + +/// Mark exit from interrupt context. Called by the arch trap handler. +pub fn exit_interrupt() { + INTERRUPT_DEPTH.fetch_sub(1, Ordering::Relaxed); +} + +/// True while the current core is handling an interrupt or exception. +pub fn in_interrupt() -> bool { + INTERRUPT_DEPTH.load(Ordering::Relaxed) > 0 +} + +/// Architecture hooks for masking interrupts on the current core. +/// `mask` masks interrupts and returns the previous interrupt state; +/// `restore` reinstates a state previously returned by `mask`. +pub struct IrqOps { + pub mask: fn() -> u64, + pub restore: fn(u64), +} + +static IRQ_OPS: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Register the architecture's mask/restore implementation. Call once +/// at early boot, before interrupts are first enabled. +pub fn set_ops(ops: &'static IrqOps) { + IRQ_OPS.store(ops as *const IrqOps as *mut IrqOps, Ordering::Release); +} + +fn ops() -> Option<&'static IrqOps> { + let ops = IRQ_OPS.load(Ordering::Acquire); + unsafe { ops.as_ref() } +} + +/// Masks interrupts on the current core for its lifetime, restoring the +/// previous mask state on drop. Nestable: taking a guard with +/// interrupts already masked (e.g. in interrupt context) is a no-op. +/// Create the guard before acquiring any lock shared with interrupt +/// context, and let it drop after the lock is released. +pub struct IrqGuard { + saved: Option, + // The saved state is core-local, so the guard must not move to + // another core: !Send + !Sync. + _not_send: PhantomData<*mut ()>, +} + +impl IrqGuard { + pub fn new() -> Self { + Self { saved: ops().map(|ops| (ops.mask)()), _not_send: PhantomData } + } +} + +impl Default for IrqGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for IrqGuard { + fn drop(&mut self) { + if let (Some(saved), Some(ops)) = (self.saved, ops()) { + (ops.restore)(saved); + } + } +} diff --git a/port/src/lib.rs b/port/src/lib.rs index d18aab3..c833ad9 100644 --- a/port/src/lib.rs +++ b/port/src/lib.rs @@ -12,6 +12,7 @@ pub mod bitmapalloc; pub mod dat; pub mod devcons; pub mod fdt; +pub mod irq; pub mod maths; pub mod mcslock; pub mod mem; diff --git a/riscv64/src/platform/virt/devcons.rs b/riscv64/src/platform/virt/devcons.rs index 557b08c..0e14df9 100644 --- a/riscv64/src/platform/virt/devcons.rs +++ b/riscv64/src/platform/virt/devcons.rs @@ -23,7 +23,7 @@ pub fn init(dt: &DeviceTree) { unsafe { let cons = &mut *CONS.get(); cons.write(uart); - Ok(cons.assume_init_mut()) + Ok(cons.assume_init_ref()) } }); } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5563919..c74d35b 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,6 +1,6 @@ [toolchain] -channel = "nightly-2026-05-20" -components = ["rustfmt", "rust-src", "clippy", "llvm-tools"] +channel = "nightly-2026-07-06" +components = ["rustfmt", "rust-analyzer", "rust-src", "clippy", "llvm-tools"] targets = [ "aarch64-unknown-none", "riscv64gc-unknown-none-elf", diff --git a/x86_64/src/devcons.rs b/x86_64/src/devcons.rs index 70c077a..c73517c 100644 --- a/x86_64/src/devcons.rs +++ b/x86_64/src/devcons.rs @@ -16,6 +16,6 @@ impl Uart for Uart16550 { pub fn init() { Console::set_uart(|| { static CONS: SyncUnsafeCell = SyncUnsafeCell::new(Uart16550 { port: 0x3f8 }); - unsafe { Ok(&mut *CONS.get()) } + unsafe { Ok(&*CONS.get()) } }); } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f29488f..c906092 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -486,13 +486,14 @@ impl QemuStep { if self.wait_for_gdb { cmd.arg("-s").arg("-S"); } - // Show exception level change events in stdout - cmd.arg("-d"); - cmd.arg("int"); cmd.arg("-kernel"); cmd.arg(format!("target/{target}/{dir}/aarch64-qemu.gz")); cmd.current_dir(workspace()); if self.verbose { + // Show exception level change events in stdout + cmd.arg("-d"); + cmd.arg("int"); + println!("Executing {cmd:?}"); } let status = annotated_status(&mut cmd)?;