diff --git a/Cargo.lock b/Cargo.lock index e55b991301..68672ad858 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,6 +1571,10 @@ dependencies = [ "toyos-wallclock", ] +[[package]] +name = "kernel-copies" +version = "0.1.0" + [[package]] name = "kernel-loom" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2018b947e3..5a057469b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "build/root", "kernel", "kernel/amdvi", + "kernel/copies", "kernel/gicv3", "kernel/its", "kernel/loom", diff --git a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md index a67353db24..41c173de17 100644 --- a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md +++ b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md @@ -6,7 +6,18 @@ opened: 2026-09-29 # Kernel forward copies and fills are one `rep movsb` or `rep stosb` on every CPU -The kernel's `memcpy`, `memset` and `memmove` are `compiler_builtins`', which +The kernel's `memcpy` is its own (`kernel/src/arch/x86_64/copy.rs`): one +`rep movsb` on every CPU, but for a copy it sends highest first. It replaced +`compiler_builtins`' `rep movsb; rep movsq; rep movsb`, so every forward copy +changed with it, on Zen 2 without ERMS too: a kernel tmpfs read of 4 or 8 KiB +took 0.92-0.96 times as long on both benches, at every distance outside the +window, and no file, IPC or pipe row slowed in both pairs (evidence +`ToyOSOrg/evidence` branch `pr-1139`, `kmemcpy/round-3`). Which copies go down +is one rule per CPUID vendor, each measured on one microarchitecture only: +AMD's window on Zen 2, Intel's none on Tiger Lake. Another AMD or Intel core +takes its vendor's rule unmeasured, until its own figures confirm or replace +it. Its `memset` +and `memmove` are `compiler_builtins`', which frame `rep movsq` or `rep stosq` with two byte strings, and take one `rep movsb` or `rep stosb` only when built with the `ermsb` target feature (`rust/library/compiler-builtins/compiler-builtins/src/mem/x86_64.rs:15-17,23-58,88-126`); @@ -19,12 +30,12 @@ and it moves the sysroot key (`RECIPE`, `build/sysroot.rs:65`). One path on ever CPU: `rep movsb` is correct without ERMS, CPUID.(7,0):EBX bit 9, and no CPU is refused. Zen 2 lacks ERMS (a Ryzen 9 PRO 3900, family 0x17, reads EBX 0x219C91A9: InstLatx64 ddff8a92, -`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`), and no proving -machine is family 0x17. +`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`); the AMD bench's +Ryzen 7 4800U is family 0x17 as well. **Exit**: a gate over `kernel.elf`, on the decoder of -`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memcpy`, -`memset` and `memmove`'s forward branch each one `rep movsb` or `rep stosb`, +`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memset` +and `memmove`'s forward branch each one `rep movsb` or `rep stosb`, with no `rep movsq` or `rep stosq`; the T14's pipe figures; and Zen 2's cost recorded, as a family-0x17 figure against the build without `ermsb` or as a `tooling` issue that owes it. **Mutation**: the build without `ermsb` reds the diff --git a/kernel/copies/Cargo.toml b/kernel/copies/Cargo.toml new file mode 100644 index 0000000000..7fac5578fa --- /dev/null +++ b/kernel/copies/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "kernel-copies" +description = "A host test of the kernel's x86-64 memcpy: its descending copy and every rule it copies by, against copy_within." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +# A package of its own because the kernel is a freestanding binary: this +# compiles only `kernel/src/arch/x86_64/copy.rs`, as `toyos-libc-copies` +# compiles libc's. diff --git a/kernel/copies/src/lib.rs b/kernel/copies/src/lib.rs new file mode 100644 index 0000000000..c1ff682771 --- /dev/null +++ b/kernel/copies/src/lib.rs @@ -0,0 +1,73 @@ +//! The kernel's x86-64 copy on an x86-64 host: the descending copy over every +//! length to 300 and some past it, at every destination offset in a store; +//! every rule's copy at every page distance below 1 KiB and some above; and +//! which copies each rule sends down. + +#[cfg(all(test, target_arch = "x86_64"))] +#[path = "../../src/arch/x86_64/copy.rs"] +mod copy; + +#[cfg(all(test, target_arch = "x86_64"))] +mod tests { + use super::copy::{self, Rule}; + + fn pattern(len: usize) -> Vec { + (0..len).map(|i| (i * 7 + i / 251) as u8).collect() + } + + /// A copy from `at` bytes past a 64-byte boundary to `distance` above + /// the next page past its end, in one buffer that ends 64 bytes past the + /// destination: the whole buffer against `copy_within`. + fn agrees(what: &str, copy: impl Fn(*mut u8, *const u8, usize), distance: usize, at: usize, n: usize) { + let gap = (at + n).next_multiple_of(4096).max(4096); + let mut got = pattern(64 + gap + distance + n + 64); + let base = got.as_ptr().align_offset(64); + let (from, to) = (base + at, base + gap + distance); + let mut want = got.clone(); + want.copy_within(from..from + n, to); + let p = got.as_mut_ptr(); + // SAFETY: both ranges are inside `got`, the destination past the source's end. + copy(unsafe { p.add(to) }, unsafe { p.add(from) }, n); + assert_eq!(got, want, "{what}: n {n} at {at} distance {distance}"); + } + + #[test] + fn the_descending_copy_agrees() { + for n in (8..300).chain([511, 512, 513, 4095, 4096, 4097, 16389]) { + for distance in 0..8 { + for at in [0, 3, 17] { + // SAFETY: `agrees` hands disjoint ranges of `n` bytes, at least 8. + agrees("copy_down", |d, s, n| unsafe { copy::copy_down(d, s, n) }, distance, at, n); + } + } + } + } + + #[test] + fn every_rule_copies_at_every_page_distance() { + let everything = Rule { window: 4096 }; + for rule in [Rule::NEVER, Rule::AMD, everything] { + for distance in (0..1024).chain([2048, 4032, 4095]) { + for n in [0, 1, 63, 64, 65, 200, 1000, 4096, 16384] { + // SAFETY: as above, `n` any length. + agrees(&format!("{rule:?}"), |d, s, n| unsafe { copy::copy_by(d, s, n, rule) }, distance, 5, n); + } + } + } + } + + #[test] + fn each_rule_sends_down_only_a_short_distance() { + let down = |rule: Rule, distance: usize, n: usize| rule.goes_down(0x10_0000 + 4096 * 3 + distance, 0x10_0000, n); + for rule in [Rule::NEVER, Rule::AMD] { + assert!(!down(rule, 0, 8192), "{rule:?}: distance 0"); + assert!(!down(rule, 16, 63), "{rule:?}: under 64 bytes"); + assert!(!down(rule, 4080, 8192), "{rule:?}: 16 bytes below modulo 4 KiB"); + } + assert!(!down(Rule::NEVER, 16, 8192)); + assert!(down(Rule::AMD, 1, 64) && down(Rule::AMD, 31, 8192) && down(Rule::AMD, 16, 64 << 20)); + assert!(!down(Rule::AMD, 32, 8192)); + // A destination a page less 16 below its source is 16 above it modulo 4 KiB. + assert!(Rule::AMD.goes_down(0x10_0000, 0x10_0000 + 4080, 4080)); + } +} diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 6de6ca6c2e..b9503a8656 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -4,7 +4,7 @@ use toyos_abi::boot::{KernelArgs, MemoryMapEntry}; -use super::{apic, control_regs, idt, ioapic, microcode, pat, percpu}; +use super::{apic, control_regs, copy, cpu, idt, ioapic, microcode, pat, percpu}; use crate::drivers::acpi::{self, MadtInfo}; use crate::log; use crate::mm::Region; @@ -64,6 +64,13 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) { microcode::init(args); microcode::load(0); + let rule = match cpu::vendor() { + kernel::cpu::Vendor::Amd => copy::Rule::AMD, + kernel::cpu::Vendor::Intel | kernel::cpu::Vendor::Other(_) => copy::Rule::NEVER, + }; + copy::set_rule(rule); + log!("memcpy: highest first {rule:?}"); + // The read-back `pat::init` owes, on a boot that now has three channels to // carry a refusal. pat::check(); diff --git a/kernel/src/arch/x86_64/copy.rs b/kernel/src/arch/x86_64/copy.rs new file mode 100644 index 0000000000..6186266068 --- /dev/null +++ b/kernel/src/arch/x86_64/copy.rs @@ -0,0 +1,156 @@ +//! The kernel's `memcpy`, in place of `compiler_builtins`' weak one: one +//! `rep movsb`, or a descending copy where a forward one would stall. +//! +//! A load matching an older in-flight store on address bits 11:0 alone waits +//! on it: AMD's Family 17h judges store-to-load forwarding by those bits (56305 +//! rev. 3.02, ยง2.12), and Intel calls it 4K aliasing (Intel 64 and IA-32 +//! Architectures Optimization Reference Manual). A forward copy whose +//! destination is `d` bytes above its source modulo 4 KiB, `0 < d < n`, loads +//! each byte `d` bytes after storing one with the same bits; copied highest +//! first, the nearest older store with those bits is 4 KiB minus `d` back. +//! The kernel touches no vector register (`build/fpgate.rs`), so the +//! descending copy moves 8 bytes a store. +//! +//! The file uses nothing but `core`, so `kernel-copies` tests it on the host; +//! only the freestanding kernel exports the symbol. + +#[cfg(target_os = "none")] +use core::sync::atomic::{AtomicUsize, Ordering}; + +/// Which copies go highest first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rule { + /// Distances below this, modulo 4 KiB, go down. + pub window: usize, +} + +/// The shortest copy that ever goes down: below it the stall is a few loads. +const LEAST: usize = 64; + +impl Rule { + /// Intel's and an unmeasured vendor's: on Tiger Lake [`copy_down`] is + /// slower than `rep movsb` at every distance and length measured. + pub const NEVER: Self = Self { window: 0 }; + + /// Zen 2's: there, `rep movsb` waits at every distance under 32 bytes, + /// at every length measured from 4 to 64 KiB, and at none from 32 up. + pub const AMD: Self = Self { window: 32 }; + + /// Whether a copy of `n` bytes from `src` to `dest` goes down. + pub fn goes_down(self, dest: usize, src: usize, n: usize) -> bool { + let d = dest.wrapping_sub(src) % 4096; + n >= LEAST && d != 0 && d < self.window + } +} + +/// The rule `memcpy` copies by, [`Rule::NEVER`] until [`set_rule`]. +#[cfg(target_os = "none")] +static WINDOW: AtomicUsize = AtomicUsize::new(0); + +/// Set the rule every later copy goes by, before another CPU runs. +#[cfg(target_os = "none")] +pub fn set_rule(rule: Rule) { + WINDOW.store(rule.window, Ordering::Relaxed); +} + +#[cfg(target_os = "none")] +fn rule() -> Rule { + Rule { window: WINDOW.load(Ordering::Relaxed) } +} + +/// Copy `n` bytes from `src` to `dest` as `rule` says. +/// +/// # Safety +/// `src` is readable and `dest` writable for `n` bytes, and the two ranges +/// are disjoint or identical, as LLVM's `memcpy` allows. +#[inline(always)] +pub unsafe fn copy_by(dest: *mut u8, src: *const u8, n: usize, rule: Rule) { + // SAFETY: as this function's caller promises; `goes_down` holds `n` to at least 8. + unsafe { + if rule.goes_down(dest as usize, src as usize, n) { + copy_down(dest, src, n); + } else { + movsb(dest, src, n); + } + } +} + +#[cfg(target_os = "none")] +#[unsafe(no_mangle)] +unsafe extern "C" fn memcpy(dest: *mut core::ffi::c_void, src: *const core::ffi::c_void, n: usize) -> *mut core::ffi::c_void { + // SAFETY: `memcpy`'s contract is `copy_by`'s. + unsafe { copy_by(dest.cast(), src.cast(), n, rule()) }; + dest +} + +/// Copy `n` bytes, lowest first. +/// +/// # Safety +/// As [`copy_by`]. +#[inline(always)] +pub unsafe fn movsb(dest: *mut u8, src: *const u8, n: usize) { + // SAFETY: as this function's caller promises; every Ring 0 entry clears DF. + unsafe { + core::arch::asm!( + "rep movsb", + inout("rdi") dest => _, + inout("rsi") src => _, + inout("rcx") n => _, + options(nostack, preserves_flags), + ); + } +} + +/// Copy `n` bytes, at least 8, between disjoint ranges, highest first: the +/// top 8 bytes in one store, then down 32 and 8 bytes a step from the +/// destination's last 8-byte boundary, so no other store splits a line, and +/// the bottom 8 bytes last. Stores past the boundary overlap one another, +/// with the same bytes. +/// +/// # Safety +/// As [`copy_by`], with `n` at least 8. +pub unsafe fn copy_down(dest: *mut u8, src: *const u8, n: usize) { + let aligned_end = ((dest as usize).wrapping_add(n) & !7).wrapping_sub(dest as usize); + // SAFETY: every access is inside `[0, n)` of its range, as the caller promises. + unsafe { + core::arch::asm!( + "mov {a}, [{s} + {n} - 8]", + "mov [{d} + {n} - 8], {a}", + "cmp {p}, 32", + "jb 3f", + "2:", + "sub {p}, 32", + "mov {a}, [{s} + {p} + 24]", + "mov {b}, [{s} + {p} + 16]", + "mov {c}, [{s} + {p} + 8]", + "mov {e}, [{s} + {p}]", + "mov [{d} + {p} + 24], {a}", + "mov [{d} + {p} + 16], {b}", + "mov [{d} + {p} + 8], {c}", + "mov [{d} + {p}], {e}", + "cmp {p}, 32", + "jae 2b", + "3:", + "cmp {p}, 8", + "jb 5f", + "4:", + "sub {p}, 8", + "mov {a}, [{s} + {p}]", + "mov [{d} + {p}], {a}", + "cmp {p}, 8", + "jae 4b", + "5:", + "mov {a}, [{s}]", + "mov [{d}], {a}", + s = in(reg) src, + d = in(reg) dest, + n = in(reg) n, + p = inout(reg) aligned_end => _, + a = out(reg) _, + b = out(reg) _, + c = out(reg) _, + e = out(reg) _, + options(nostack), + ); + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index 4fccc30d21..fe6fddb229 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -29,6 +29,7 @@ pub mod boot; pub mod cache; pub mod clocksource; pub mod control_regs; +pub mod copy; pub mod counters; pub mod console_uart; pub mod cpu;