From 12b7c23f49ea8c5dc69031d6d842df49e0e207d2 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 00:06:28 +0100 Subject: [PATCH 1/3] kernel: memcpy goes highest first where a forward copy would stall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A load whose address bits 11:0 match an older in-flight store waits on it: AMD's Family 17h judges store-to-load forwarding by those bits alone (56305 rev. 3.02, §2.12), and Intel documents the same as 4K aliasing. A forward string copy whose destination sits a short way above its source modulo 4 KiB stalls on every load: userland's `rep movsb` over 8 KiB at 16 bytes took 2605 ns against 90 at 0 on the AMD bench (Ryzen 7 4800U, Zen 2), and on the T14 (i5-1135G7, Tiger Lake) 750 ns from 16 to 64 bytes, 191 at 256, against 62 at 0 (osbench copyat, evidence pr-1133). The kernel's memcpy was compiler_builtins' `rep movsb; rep movsq; rep movsb`, the sysroot being built without `ermsb`; the A/B in the commits after this one measures what that sequence and this one each cost. The kernel does hit those distances. A measurement-only probe of every kernel memcpy under QEMU, by distance and caller, found osbench's file group (the kernel's tmpfs) moving 12.3% of its bytes at 1-511 bytes and 1.3% at 1-63, through the user copies' pieces, about what uniformly placed buffers give; the pipe and IPC groups' buffers sat clear of them. The kernel now defines memcpy itself, taking the place of compiler_builtins' weak symbol: a copy of at least 64 bytes whose distance is under the CPU's window and under its length goes down by general registers, 8 bytes a store, since the kernel touches no vector register; every other copy is one `rep movsb`. That changes every forward copy outside the window too, from the qword sequence to the byte string, on every CPU including Zen 2, which has no ERMS. The window is the CPU's, estimated here and set by that A/B: 32 bytes on Zen 2, and 192 up to 16 KiB on Tiger Lake. The rule is chosen once by CPUID's vendor at boot; another vendor's copies never go down. kernel-copies tests the copy on an x86-64 host against copy_within, and which copies each rule sends down. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 4 + Cargo.toml | 1 + ...are-one-rep-movsb-or-stosb-on-every-cpu.md | 8 +- kernel/copies/Cargo.toml | 11 ++ kernel/copies/src/lib.rs | 76 +++++++++ kernel/src/arch/x86_64/boot.rs | 10 +- kernel/src/arch/x86_64/copy.rs | 158 ++++++++++++++++++ kernel/src/arch/x86_64/mod.rs | 1 + 8 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 kernel/copies/Cargo.toml create mode 100644 kernel/copies/src/lib.rs create mode 100644 kernel/src/arch/x86_64/copy.rs diff --git a/Cargo.lock b/Cargo.lock index e55b9913015..68672ad8580 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,6 +1571,10 @@ dependencies = [ "toyos-wallclock", ] +[[package]] +name = "kernel-copies" +version = "0.1.0" + [[package]] name = "kernel-loom" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2018b947e34..5a057469b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "build/root", "kernel", "kernel/amdvi", + "kernel/copies", "kernel/gicv3", "kernel/its", "kernel/loom", diff --git a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md index a67353db24d..9e471ef0c15 100644 --- a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md +++ b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md @@ -6,7 +6,9 @@ opened: 2026-09-29 # Kernel forward copies and fills are one `rep movsb` or `rep stosb` on every CPU -The kernel's `memcpy`, `memset` and `memmove` are `compiler_builtins`', which +The kernel's `memcpy` is its own (`kernel/src/arch/x86_64/copy.rs`): one +`rep movsb` on every CPU, but for a copy it sends highest first. Its `memset` +and `memmove` are `compiler_builtins`', which frame `rep movsq` or `rep stosq` with two byte strings, and take one `rep movsb` or `rep stosb` only when built with the `ermsb` target feature (`rust/library/compiler-builtins/compiler-builtins/src/mem/x86_64.rs:15-17,23-58,88-126`); @@ -23,8 +25,8 @@ refused. Zen 2 lacks ERMS (a Ryzen 9 PRO 3900, family 0x17, reads EBX machine is family 0x17. **Exit**: a gate over `kernel.elf`, on the decoder of -`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memcpy`, -`memset` and `memmove`'s forward branch each one `rep movsb` or `rep stosb`, +`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memset` +and `memmove`'s forward branch each one `rep movsb` or `rep stosb`, with no `rep movsq` or `rep stosq`; the T14's pipe figures; and Zen 2's cost recorded, as a family-0x17 figure against the build without `ermsb` or as a `tooling` issue that owes it. **Mutation**: the build without `ermsb` reds the diff --git a/kernel/copies/Cargo.toml b/kernel/copies/Cargo.toml new file mode 100644 index 00000000000..7fac5578fa9 --- /dev/null +++ b/kernel/copies/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "kernel-copies" +description = "A host test of the kernel's x86-64 memcpy: its descending copy and every rule it copies by, against copy_within." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +# A package of its own because the kernel is a freestanding binary: this +# compiles only `kernel/src/arch/x86_64/copy.rs`, as `toyos-libc-copies` +# compiles libc's. diff --git a/kernel/copies/src/lib.rs b/kernel/copies/src/lib.rs new file mode 100644 index 00000000000..e4d3feb20a5 --- /dev/null +++ b/kernel/copies/src/lib.rs @@ -0,0 +1,76 @@ +//! The kernel's x86-64 copy on an x86-64 host: the descending copy over every +//! length to 300 and some past it, at every destination offset in a store; +//! every rule's copy at every page distance below 1 KiB and some above; and +//! which copies each rule sends down. + +#[cfg(all(test, target_arch = "x86_64"))] +#[path = "../../src/arch/x86_64/copy.rs"] +mod copy; + +#[cfg(all(test, target_arch = "x86_64"))] +mod tests { + use super::copy::{self, Rule}; + + fn pattern(len: usize) -> Vec { + (0..len).map(|i| (i * 7 + i / 251) as u8).collect() + } + + /// A copy from `at` bytes past a 64-byte boundary to `distance` above + /// the next page past its end, in one buffer that ends 64 bytes past the + /// destination: the whole buffer against `copy_within`. + fn agrees(what: &str, copy: impl Fn(*mut u8, *const u8, usize), distance: usize, at: usize, n: usize) { + let gap = (at + n).next_multiple_of(4096).max(4096); + let mut got = pattern(64 + gap + distance + n + 64); + let base = got.as_ptr().align_offset(64); + let (from, to) = (base + at, base + gap + distance); + let mut want = got.clone(); + want.copy_within(from..from + n, to); + let p = got.as_mut_ptr(); + // SAFETY: both ranges are inside `got`, the destination past the source's end. + copy(unsafe { p.add(to) }, unsafe { p.add(from) }, n); + assert_eq!(got, want, "{what}: n {n} at {at} distance {distance}"); + } + + #[test] + fn the_descending_copy_agrees() { + for n in (8..300).chain([511, 512, 513, 4095, 4096, 4097, 16389]) { + for distance in 0..8 { + for at in [0, 3, 17] { + // SAFETY: `agrees` hands disjoint ranges of `n` bytes, at least 8. + agrees("copy_down", |d, s, n| unsafe { copy::copy_down(d, s, n) }, distance, at, n); + } + } + } + } + + #[test] + fn every_rule_copies_at_every_page_distance() { + let everything = Rule { window: 4096, most: usize::MAX }; + for rule in [Rule::NEVER, Rule::AMD, Rule::INTEL, everything] { + for distance in (0..1024).chain([2048, 4032, 4095]) { + for n in [0, 1, 63, 64, 65, 200, 1000, 4096, 16384] { + // SAFETY: as above, `n` any length. + agrees(&format!("{rule:?}"), |d, s, n| unsafe { copy::copy_by(d, s, n, rule) }, distance, 5, n); + } + } + } + } + + #[test] + fn each_rule_sends_down_only_a_short_distance_closer_than_the_copy_is_long() { + let down = |rule: Rule, distance: usize, n: usize| rule.goes_down(0x10_0000 + 4096 * 3 + distance, 0x10_0000, n); + for rule in [Rule::NEVER, Rule::AMD, Rule::INTEL] { + assert!(!down(rule, 0, 8192), "{rule:?}: distance 0"); + assert!(!down(rule, 16, 63), "{rule:?}: under 64 bytes"); + assert!(!down(rule, 100, 100), "{rule:?}: a distance as long as the copy"); + assert!(!down(rule, 4080, 8192), "{rule:?}: 16 bytes below modulo 4 KiB"); + } + assert!(!down(Rule::NEVER, 16, 8192)); + assert!(down(Rule::AMD, 1, 64) && down(Rule::AMD, 31, 8192) && down(Rule::AMD, 16, 64 << 20)); + assert!(!down(Rule::AMD, 32, 8192)); + assert!(down(Rule::INTEL, 16, 4096) && down(Rule::INTEL, 191, 16384) && down(Rule::INTEL, 99, 100)); + assert!(!down(Rule::INTEL, 192, 8192) && !down(Rule::INTEL, 16, 16385)); + // A destination a page less 16 below its source is 16 above it modulo 4 KiB. + assert!(Rule::AMD.goes_down(0x10_0000, 0x10_0000 + 4080, 4080)); + } +} diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 6de6ca6c2ee..2820fbfe682 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -4,7 +4,7 @@ use toyos_abi::boot::{KernelArgs, MemoryMapEntry}; -use super::{apic, control_regs, idt, ioapic, microcode, pat, percpu}; +use super::{apic, control_regs, copy, cpu, idt, ioapic, microcode, pat, percpu}; use crate::drivers::acpi::{self, MadtInfo}; use crate::log; use crate::mm::Region; @@ -64,6 +64,14 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) { microcode::init(args); microcode::load(0); + let rule = match cpu::vendor() { + kernel::cpu::Vendor::Intel => copy::Rule::INTEL, + kernel::cpu::Vendor::Amd => copy::Rule::AMD, + kernel::cpu::Vendor::Other(_) => copy::Rule::NEVER, + }; + copy::set_rule(rule); + log!("memcpy: highest first {rule:?}"); + // The read-back `pat::init` owes, on a boot that now has three channels to // carry a refusal. pat::check(); diff --git a/kernel/src/arch/x86_64/copy.rs b/kernel/src/arch/x86_64/copy.rs new file mode 100644 index 00000000000..96d61dedd5c --- /dev/null +++ b/kernel/src/arch/x86_64/copy.rs @@ -0,0 +1,158 @@ +//! The kernel's `memcpy`, in place of `compiler_builtins`' weak one: one +//! `rep movsb`, or a descending copy where a forward one would stall. +//! +//! A load matching an older in-flight store on address bits 11:0 alone waits +//! on it: AMD's Family 17h judges store-to-load forwarding by those bits (56305 +//! rev. 3.02, §2.12), and Intel calls it 4K aliasing (Intel 64 and IA-32 +//! Architectures Optimization Reference Manual). A forward copy whose +//! destination is `d` bytes above its source modulo 4 KiB, `0 < d < n`, loads +//! each byte `d` bytes after storing one with the same bits; copied highest +//! first, that store comes 4 KiB minus `d` later. The kernel touches no vector +//! register (`build/fpgate.rs`), so the descending copy moves 8 bytes a store. +//! +//! The file uses nothing but `core`, so `kernel-copies` tests it on the host; +//! only the freestanding kernel exports the symbol. + +use core::sync::atomic::{AtomicUsize, Ordering}; + +/// Which copies go highest first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rule { + /// Distances below this, modulo 4 KiB, go down. + pub window: usize, + /// The longest copy that does. + pub most: usize, +} + +/// The shortest copy that ever goes down: below it the stall is a few loads. +const LEAST: usize = 64; + +impl Rule { + pub const NEVER: Self = Self { window: 0, most: 0 }; + + /// Zen 2 loads and stores 32 bytes at a time (56305 §2.12), and only a + /// load overlapping the store before it in part waits, at every length. + pub const AMD: Self = Self { window: 32, most: usize::MAX }; + + /// Tiger Lake's loads wait on stores hundreds of bytes back, less the + /// farther; [`copy_down`], a store a cycle, beats `rep movsb` under 192. + /// Past its L1 cache the wait fades, and the fast string, which need not + /// read a line it writes whole, outruns [`copy_down`]. + pub const INTEL: Self = Self { window: 192, most: 16 * 1024 }; + + /// Whether a copy of `n` bytes from `src` to `dest` goes down. + pub fn goes_down(self, dest: usize, src: usize, n: usize) -> bool { + let d = dest.wrapping_sub(src) % 4096; + (LEAST..=self.most).contains(&n) && d != 0 && d < self.window.min(n) + } +} + +/// The rule `memcpy` copies by, [`Rule::NEVER`] until [`set_rule`]. +static WINDOW: AtomicUsize = AtomicUsize::new(0); +static MOST: AtomicUsize = AtomicUsize::new(0); + +/// Set the rule every later copy goes by, before another CPU runs. +pub fn set_rule(rule: Rule) { + WINDOW.store(rule.window, Ordering::Relaxed); + MOST.store(rule.most, Ordering::Relaxed); +} + +fn rule() -> Rule { + Rule { window: WINDOW.load(Ordering::Relaxed), most: MOST.load(Ordering::Relaxed) } +} + +/// Copy `n` bytes from `src` to `dest`, which do not overlap, as `rule` says. +/// +/// # Safety +/// `src` is readable and `dest` writable for `n` bytes, and the two do not overlap. +#[inline(always)] +pub unsafe fn copy_by(dest: *mut u8, src: *const u8, n: usize, rule: Rule) { + // SAFETY: as this function's caller promises; `goes_down` holds `n` to at least 8. + unsafe { + if rule.goes_down(dest as usize, src as usize, n) { + copy_down(dest, src, n); + } else { + movsb(dest, src, n); + } + } +} + +#[cfg(target_os = "none")] +#[unsafe(no_mangle)] +unsafe extern "C" fn memcpy(dest: *mut core::ffi::c_void, src: *const core::ffi::c_void, n: usize) -> *mut core::ffi::c_void { + // SAFETY: `memcpy`'s contract is `copy_by`'s. + unsafe { copy_by(dest.cast(), src.cast(), n, rule()) }; + dest +} + +/// Copy `n` bytes, lowest first. +/// +/// # Safety +/// As [`copy_by`]. +#[inline(always)] +pub unsafe fn movsb(dest: *mut u8, src: *const u8, n: usize) { + // SAFETY: as this function's caller promises; every Ring 0 entry clears DF. + unsafe { + core::arch::asm!( + "rep movsb", + inout("rdi") dest => _, + inout("rsi") src => _, + inout("rcx") n => _, + options(nostack, preserves_flags), + ); + } +} + +/// Copy `n` bytes, at least 8, between disjoint ranges, highest first: the +/// top 8 bytes in one store, then down 32 and 8 bytes a step from the +/// destination's last 8-byte boundary, so no other store splits a line, and +/// the bottom 8 bytes last. Stores past the boundary overlap one another, +/// with the same bytes. +/// +/// # Safety +/// As [`copy_by`], with `n` at least 8. +pub unsafe fn copy_down(dest: *mut u8, src: *const u8, n: usize) { + let aligned_end = ((dest as usize).wrapping_add(n) & !7).wrapping_sub(dest as usize); + // SAFETY: every access is inside `[0, n)` of its range, as the caller promises. + unsafe { + core::arch::asm!( + "mov {a}, [{s} + {n} - 8]", + "mov [{d} + {n} - 8], {a}", + "cmp {p}, 32", + "jb 3f", + "2:", + "sub {p}, 32", + "mov {a}, [{s} + {p} + 24]", + "mov {b}, [{s} + {p} + 16]", + "mov {c}, [{s} + {p} + 8]", + "mov {e}, [{s} + {p}]", + "mov [{d} + {p} + 24], {a}", + "mov [{d} + {p} + 16], {b}", + "mov [{d} + {p} + 8], {c}", + "mov [{d} + {p}], {e}", + "cmp {p}, 32", + "jae 2b", + "3:", + "cmp {p}, 8", + "jb 5f", + "4:", + "sub {p}, 8", + "mov {a}, [{s} + {p}]", + "mov [{d} + {p}], {a}", + "cmp {p}, 8", + "jae 4b", + "5:", + "mov {a}, [{s}]", + "mov [{d}], {a}", + s = in(reg) src, + d = in(reg) dest, + n = in(reg) n, + p = inout(reg) aligned_end => _, + a = out(reg) _, + b = out(reg) _, + c = out(reg) _, + e = out(reg) _, + options(nostack), + ); + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index 4fccc30d21d..fe6fddb2298 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -29,6 +29,7 @@ pub mod boot; pub mod cache; pub mod clocksource; pub mod control_regs; +pub mod copy; pub mod counters; pub mod console_uart; pub mod cpu; From 31586ef088f05da492dd8cbbe79a4ae00bcd5c65 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 00:06:28 +0100 Subject: [PATCH 2/3] review r1 of #1139: the rule's state only in the kernel, and the contracts as they are - The rule's statics, set_rule and rule() exist only where memcpy does (target_os = "none"), so kernel-copies builds without dead code on an x86-64 host under -D warnings. - copy_by's contract is disjoint or identical ranges, as LLVM's memcpy allows. - The module header: copied highest first, the nearest older store with a load's bits 11:0 is 4 KiB less the distance back, not ahead. - The track issue records that memcpy replaced compiler_builtins' qword sequence on every forward copy, that each vendor's rule is measured on one microarchitecture, and that the AMD bench is family 0x17. Co-Authored-By: Claude Opus 5.5 --- ...lls-are-one-rep-movsb-or-stosb-on-every-cpu.md | 11 ++++++++--- kernel/src/arch/x86_64/copy.rs | 15 +++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md index 9e471ef0c15..378d685bce6 100644 --- a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md +++ b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md @@ -7,7 +7,12 @@ opened: 2026-09-29 # Kernel forward copies and fills are one `rep movsb` or `rep stosb` on every CPU The kernel's `memcpy` is its own (`kernel/src/arch/x86_64/copy.rs`): one -`rep movsb` on every CPU, but for a copy it sends highest first. Its `memset` +`rep movsb` on every CPU, but for a copy it sends highest first. It replaced +`compiler_builtins`' `rep movsb; rep movsq; rep movsb`, so every forward copy +changed with it, on Zen 2 without ERMS too. Which copies go down is one rule +per CPUID vendor, each measured on one microarchitecture only: AMD's on Zen 2, +Intel's on Tiger Lake. Another AMD or Intel core takes its vendor's rule +unmeasured, until its own figures confirm or replace it. Its `memset` and `memmove` are `compiler_builtins`', which frame `rep movsq` or `rep stosq` with two byte strings, and take one `rep movsb` or `rep stosb` only when built with the `ermsb` target feature @@ -21,8 +26,8 @@ and it moves the sysroot key (`RECIPE`, `build/sysroot.rs:65`). One path on ever CPU: `rep movsb` is correct without ERMS, CPUID.(7,0):EBX bit 9, and no CPU is refused. Zen 2 lacks ERMS (a Ryzen 9 PRO 3900, family 0x17, reads EBX 0x219C91A9: InstLatx64 ddff8a92, -`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`), and no proving -machine is family 0x17. +`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`); the AMD bench's +Ryzen 7 4800U is family 0x17 as well. **Exit**: a gate over `kernel.elf`, on the decoder of `issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memset` diff --git a/kernel/src/arch/x86_64/copy.rs b/kernel/src/arch/x86_64/copy.rs index 96d61dedd5c..ca95488f5be 100644 --- a/kernel/src/arch/x86_64/copy.rs +++ b/kernel/src/arch/x86_64/copy.rs @@ -7,12 +7,14 @@ //! Architectures Optimization Reference Manual). A forward copy whose //! destination is `d` bytes above its source modulo 4 KiB, `0 < d < n`, loads //! each byte `d` bytes after storing one with the same bits; copied highest -//! first, that store comes 4 KiB minus `d` later. The kernel touches no vector -//! register (`build/fpgate.rs`), so the descending copy moves 8 bytes a store. +//! first, the nearest older store with those bits is 4 KiB minus `d` back. +//! The kernel touches no vector register (`build/fpgate.rs`), so the +//! descending copy moves 8 bytes a store. //! //! The file uses nothing but `core`, so `kernel-copies` tests it on the host; //! only the freestanding kernel exports the symbol. +#[cfg(target_os = "none")] use core::sync::atomic::{AtomicUsize, Ordering}; /// Which copies go highest first. @@ -48,23 +50,28 @@ impl Rule { } /// The rule `memcpy` copies by, [`Rule::NEVER`] until [`set_rule`]. +#[cfg(target_os = "none")] static WINDOW: AtomicUsize = AtomicUsize::new(0); +#[cfg(target_os = "none")] static MOST: AtomicUsize = AtomicUsize::new(0); /// Set the rule every later copy goes by, before another CPU runs. +#[cfg(target_os = "none")] pub fn set_rule(rule: Rule) { WINDOW.store(rule.window, Ordering::Relaxed); MOST.store(rule.most, Ordering::Relaxed); } +#[cfg(target_os = "none")] fn rule() -> Rule { Rule { window: WINDOW.load(Ordering::Relaxed), most: MOST.load(Ordering::Relaxed) } } -/// Copy `n` bytes from `src` to `dest`, which do not overlap, as `rule` says. +/// Copy `n` bytes from `src` to `dest` as `rule` says. /// /// # Safety -/// `src` is readable and `dest` writable for `n` bytes, and the two do not overlap. +/// `src` is readable and `dest` writable for `n` bytes, and the two ranges +/// are disjoint or identical, as LLVM's `memcpy` allows. #[inline(always)] pub unsafe fn copy_by(dest: *mut u8, src: *const u8, n: usize, rule: Rule) { // SAFETY: as this function's caller promises; `goes_down` holds `n` to at least 8. From c020b27f2a5d9a33d30298a18137c0138b020c68 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 00:06:44 +0100 Subject: [PATCH 3/3] A/B of #1139: only AMD copies go down, by a window alone Four boots per bench (base, head, head, base) of base b49e6aec4 (main and osbench kcopyat) against head dc535330f (base and this change at its first review). Evidence: ToyOSOrg/evidence branch pr-1139, kmemcpy/round-3. - Zen 2 (AMD bench, Ryzen 7 4800U): in this process, `rep movsb` waits at every distance under 32 bytes and at every length from 4 to 64 KiB (8 KiB: 1946 ns, 64 KiB: 20463 ns, against 88 and 962 at 0), and at none from 32 bytes up. The descending copy takes 260 ns at 8 KiB and 2009 ns at 64 KiB at every distance. Window 32, at every length, stays. - Tiger Lake (T14, i5-1135G7): the descending copy is slower than `rep movsb` at every distance and length measured (8 KiB: 241-256 ns against 58-197), so Intel's copies never go down. With one window left and no cap, the length cap and the distance-under-length test can no longer decide anything, and go. - The forward path, compiler_builtins' `rep movsb; rep movsq; rep movsb` replaced by one `rep movsb`, is a gain on both benches: a kernel tmpfs read of 4 or 8 KiB took 0.92-0.96 times base's at d = 0 and at every distance outside the window. No file, IPC or pipe row was slower in both pairs; the file metadata rows took 0.71-0.93 times as long. - The issue's exit holds: a kernel read 16 bytes above its source takes 1.17 (AMD) and 1.10 (T14) times the same read at 0 for 8 KiB. - Base did not stall at d < 32 on the AMD: its qword string did not wait there, where `rep movsb` does; the window keeps the byte string from it. Co-Authored-By: Claude Opus 5.5 --- ...are-one-rep-movsb-or-stosb-on-every-cpu.md | 12 ++++++--- kernel/copies/src/lib.rs | 11 +++----- kernel/src/arch/x86_64/boot.rs | 3 +-- kernel/src/arch/x86_64/copy.rs | 25 ++++++------------- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md index 378d685bce6..41c173de174 100644 --- a/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md +++ b/issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md @@ -9,10 +9,14 @@ opened: 2026-09-29 The kernel's `memcpy` is its own (`kernel/src/arch/x86_64/copy.rs`): one `rep movsb` on every CPU, but for a copy it sends highest first. It replaced `compiler_builtins`' `rep movsb; rep movsq; rep movsb`, so every forward copy -changed with it, on Zen 2 without ERMS too. Which copies go down is one rule -per CPUID vendor, each measured on one microarchitecture only: AMD's on Zen 2, -Intel's on Tiger Lake. Another AMD or Intel core takes its vendor's rule -unmeasured, until its own figures confirm or replace it. Its `memset` +changed with it, on Zen 2 without ERMS too: a kernel tmpfs read of 4 or 8 KiB +took 0.92-0.96 times as long on both benches, at every distance outside the +window, and no file, IPC or pipe row slowed in both pairs (evidence +`ToyOSOrg/evidence` branch `pr-1139`, `kmemcpy/round-3`). Which copies go down +is one rule per CPUID vendor, each measured on one microarchitecture only: +AMD's window on Zen 2, Intel's none on Tiger Lake. Another AMD or Intel core +takes its vendor's rule unmeasured, until its own figures confirm or replace +it. Its `memset` and `memmove` are `compiler_builtins`', which frame `rep movsq` or `rep stosq` with two byte strings, and take one `rep movsb` or `rep stosb` only when built with the `ermsb` target feature diff --git a/kernel/copies/src/lib.rs b/kernel/copies/src/lib.rs index e4d3feb20a5..c1ff6827718 100644 --- a/kernel/copies/src/lib.rs +++ b/kernel/copies/src/lib.rs @@ -45,8 +45,8 @@ mod tests { #[test] fn every_rule_copies_at_every_page_distance() { - let everything = Rule { window: 4096, most: usize::MAX }; - for rule in [Rule::NEVER, Rule::AMD, Rule::INTEL, everything] { + let everything = Rule { window: 4096 }; + for rule in [Rule::NEVER, Rule::AMD, everything] { for distance in (0..1024).chain([2048, 4032, 4095]) { for n in [0, 1, 63, 64, 65, 200, 1000, 4096, 16384] { // SAFETY: as above, `n` any length. @@ -57,19 +57,16 @@ mod tests { } #[test] - fn each_rule_sends_down_only_a_short_distance_closer_than_the_copy_is_long() { + fn each_rule_sends_down_only_a_short_distance() { let down = |rule: Rule, distance: usize, n: usize| rule.goes_down(0x10_0000 + 4096 * 3 + distance, 0x10_0000, n); - for rule in [Rule::NEVER, Rule::AMD, Rule::INTEL] { + for rule in [Rule::NEVER, Rule::AMD] { assert!(!down(rule, 0, 8192), "{rule:?}: distance 0"); assert!(!down(rule, 16, 63), "{rule:?}: under 64 bytes"); - assert!(!down(rule, 100, 100), "{rule:?}: a distance as long as the copy"); assert!(!down(rule, 4080, 8192), "{rule:?}: 16 bytes below modulo 4 KiB"); } assert!(!down(Rule::NEVER, 16, 8192)); assert!(down(Rule::AMD, 1, 64) && down(Rule::AMD, 31, 8192) && down(Rule::AMD, 16, 64 << 20)); assert!(!down(Rule::AMD, 32, 8192)); - assert!(down(Rule::INTEL, 16, 4096) && down(Rule::INTEL, 191, 16384) && down(Rule::INTEL, 99, 100)); - assert!(!down(Rule::INTEL, 192, 8192) && !down(Rule::INTEL, 16, 16385)); // A destination a page less 16 below its source is 16 above it modulo 4 KiB. assert!(Rule::AMD.goes_down(0x10_0000, 0x10_0000 + 4080, 4080)); } diff --git a/kernel/src/arch/x86_64/boot.rs b/kernel/src/arch/x86_64/boot.rs index 2820fbfe682..b9503a86560 100644 --- a/kernel/src/arch/x86_64/boot.rs +++ b/kernel/src/arch/x86_64/boot.rs @@ -65,9 +65,8 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) { microcode::load(0); let rule = match cpu::vendor() { - kernel::cpu::Vendor::Intel => copy::Rule::INTEL, kernel::cpu::Vendor::Amd => copy::Rule::AMD, - kernel::cpu::Vendor::Other(_) => copy::Rule::NEVER, + kernel::cpu::Vendor::Intel | kernel::cpu::Vendor::Other(_) => copy::Rule::NEVER, }; copy::set_rule(rule); log!("memcpy: highest first {rule:?}"); diff --git a/kernel/src/arch/x86_64/copy.rs b/kernel/src/arch/x86_64/copy.rs index ca95488f5be..6186266068a 100644 --- a/kernel/src/arch/x86_64/copy.rs +++ b/kernel/src/arch/x86_64/copy.rs @@ -22,49 +22,40 @@ use core::sync::atomic::{AtomicUsize, Ordering}; pub struct Rule { /// Distances below this, modulo 4 KiB, go down. pub window: usize, - /// The longest copy that does. - pub most: usize, } /// The shortest copy that ever goes down: below it the stall is a few loads. const LEAST: usize = 64; impl Rule { - pub const NEVER: Self = Self { window: 0, most: 0 }; + /// Intel's and an unmeasured vendor's: on Tiger Lake [`copy_down`] is + /// slower than `rep movsb` at every distance and length measured. + pub const NEVER: Self = Self { window: 0 }; - /// Zen 2 loads and stores 32 bytes at a time (56305 §2.12), and only a - /// load overlapping the store before it in part waits, at every length. - pub const AMD: Self = Self { window: 32, most: usize::MAX }; - - /// Tiger Lake's loads wait on stores hundreds of bytes back, less the - /// farther; [`copy_down`], a store a cycle, beats `rep movsb` under 192. - /// Past its L1 cache the wait fades, and the fast string, which need not - /// read a line it writes whole, outruns [`copy_down`]. - pub const INTEL: Self = Self { window: 192, most: 16 * 1024 }; + /// Zen 2's: there, `rep movsb` waits at every distance under 32 bytes, + /// at every length measured from 4 to 64 KiB, and at none from 32 up. + pub const AMD: Self = Self { window: 32 }; /// Whether a copy of `n` bytes from `src` to `dest` goes down. pub fn goes_down(self, dest: usize, src: usize, n: usize) -> bool { let d = dest.wrapping_sub(src) % 4096; - (LEAST..=self.most).contains(&n) && d != 0 && d < self.window.min(n) + n >= LEAST && d != 0 && d < self.window } } /// The rule `memcpy` copies by, [`Rule::NEVER`] until [`set_rule`]. #[cfg(target_os = "none")] static WINDOW: AtomicUsize = AtomicUsize::new(0); -#[cfg(target_os = "none")] -static MOST: AtomicUsize = AtomicUsize::new(0); /// Set the rule every later copy goes by, before another CPU runs. #[cfg(target_os = "none")] pub fn set_rule(rule: Rule) { WINDOW.store(rule.window, Ordering::Relaxed); - MOST.store(rule.most, Ordering::Relaxed); } #[cfg(target_os = "none")] fn rule() -> Rule { - Rule { window: WINDOW.load(Ordering::Relaxed), most: MOST.load(Ordering::Relaxed) } + Rule { window: WINDOW.load(Ordering::Relaxed) } } /// Copy `n` bytes from `src` to `dest` as `rule` says.