kernel: memcpy goes highest first where a forward copy would stall

Cargo.toml

@@ -25,6 +25,7 @@ members = [
     "build/root",
     "kernel",
     "kernel/amdvi",
+    "kernel/copies",
     "kernel/gicv3",
     "kernel/its",
     "kernel/loom",

issues/kernel-forward-copies-and-fills-are-one-rep-movsb-or-stosb-on-every-cpu.md

@@ -6,7 +6,18 @@ opened: 2026-09-29
 
 # Kernel forward copies and fills are one `rep movsb` or `rep stosb` on every CPU
 
-The kernel's `memcpy`, `memset` and `memmove` are `compiler_builtins`', which
+The kernel's `memcpy` is its own (`kernel/src/arch/x86_64/copy.rs`): one
+`rep movsb` on every CPU, but for a copy it sends highest first. It replaced
+`compiler_builtins`' `rep movsb; rep movsq; rep movsb`, so every forward copy
+changed with it, on Zen 2 without ERMS too: a kernel tmpfs read of 4 or 8 KiB
+took 0.92-0.96 times as long on both benches, at every distance outside the
+window, and no file, IPC or pipe row slowed in both pairs (evidence
+`ToyOSOrg/evidence` branch `pr-1139`, `kmemcpy/round-3`). Which copies go down
+is one rule per CPUID vendor, each measured on one microarchitecture only:
+AMD's window on Zen 2, Intel's none on Tiger Lake. Another AMD or Intel core
+takes its vendor's rule unmeasured, until its own figures confirm or replace
+it. Its `memset`
+and `memmove` are `compiler_builtins`', which
 frame `rep movsq` or `rep stosq` with two byte strings, and take one `rep movsb`
 or `rep stosb` only when built with the `ermsb` target feature
 (`rust/library/compiler-builtins/compiler-builtins/src/mem/x86_64.rs:15-17,23-58,88-126`);
@@ -19,12 +30,12 @@ and it moves the sysroot key (`RECIPE`, `build/sysroot.rs:65`). One path on ever
 CPU: `rep movsb` is correct without ERMS, CPUID.(7,0):EBX bit 9, and no CPU is
 refused. Zen 2 lacks ERMS (a Ryzen 9 PRO 3900, family 0x17, reads EBX
 0x219C91A9: InstLatx64 ddff8a92,
-`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`), and no proving
-machine is family 0x17.
+`AuthenticAMD/AuthenticAMD0870F10_K17_Matisse_CPUID.txt:52`); the AMD bench's
+Ryzen 7 4800U is family 0x17 as well.
 
 **Exit**: a gate over `kernel.elf`, on the decoder of
-`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memcpy`,
-`memset` and `memmove`'s forward branch each one `rep movsb` or `rep stosb`,
+`issues/no-gate-decodes-kernel-elfs-instructions.md`, finds `memset`
+and `memmove`'s forward branch each one `rep movsb` or `rep stosb`,
 with no `rep movsq` or `rep stosq`; the T14's pipe figures; and Zen 2's cost
 recorded, as a family-0x17 figure against the build without `ermsb` or as a
 `tooling` issue that owes it. **Mutation**: the build without `ermsb` reds the

kernel/copies/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "kernel-copies"
+description = "A host test of the kernel's x86-64 memcpy: its descending copy and every rule it copies by, against copy_within."
+version = "0.1.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+publish = false
+
+# A package of its own because the kernel is a freestanding binary: this
+# compiles only `kernel/src/arch/x86_64/copy.rs`, as `toyos-libc-copies`
+# compiles libc's.

kernel/copies/src/lib.rs

@@ -0,0 +1,73 @@
+//! The kernel's x86-64 copy on an x86-64 host: the descending copy over every
+//! length to 300 and some past it, at every destination offset in a store;
+//! every rule's copy at every page distance below 1 KiB and some above; and
+//! which copies each rule sends down.
+
+#[cfg(all(test, target_arch = "x86_64"))]
+#[path = "../../src/arch/x86_64/copy.rs"]
+mod copy;
+
+#[cfg(all(test, target_arch = "x86_64"))]
+mod tests {
+    use super::copy::{self, Rule};
+
+    fn pattern(len: usize) -> Vec<u8> {
+        (0..len).map(|i| (i * 7 + i / 251) as u8).collect()
+    }
+
+    /// A copy from `at` bytes past a 64-byte boundary to `distance` above
+    /// the next page past its end, in one buffer that ends 64 bytes past the
+    /// destination: the whole buffer against `copy_within`.
+    fn agrees(what: &str, copy: impl Fn(*mut u8, *const u8, usize), distance: usize, at: usize, n: usize) {
+        let gap = (at + n).next_multiple_of(4096).max(4096);
+        let mut got = pattern(64 + gap + distance + n + 64);
+        let base = got.as_ptr().align_offset(64);
+        let (from, to) = (base + at, base + gap + distance);
+        let mut want = got.clone();
+        want.copy_within(from..from + n, to);
+        let p = got.as_mut_ptr();
+        // SAFETY: both ranges are inside `got`, the destination past the source's end.
+        copy(unsafe { p.add(to) }, unsafe { p.add(from) }, n);
+        assert_eq!(got, want, "{what}: n {n} at {at} distance {distance}");
+    }
+
+    #[test]
+    fn the_descending_copy_agrees() {
+        for n in (8..300).chain([511, 512, 513, 4095, 4096, 4097, 16389]) {
+            for distance in 0..8 {
+                for at in [0, 3, 17] {
+                    // SAFETY: `agrees` hands disjoint ranges of `n` bytes, at least 8.
+                    agrees("copy_down", |d, s, n| unsafe { copy::copy_down(d, s, n) }, distance, at, n);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn every_rule_copies_at_every_page_distance() {
+        let everything = Rule { window: 4096 };
+        for rule in [Rule::NEVER, Rule::AMD, everything] {
+            for distance in (0..1024).chain([2048, 4032, 4095]) {
+                for n in [0, 1, 63, 64, 65, 200, 1000, 4096, 16384] {
+                    // SAFETY: as above, `n` any length.
+                    agrees(&format!("{rule:?}"), |d, s, n| unsafe { copy::copy_by(d, s, n, rule) }, distance, 5, n);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn each_rule_sends_down_only_a_short_distance() {
+        let down = |rule: Rule, distance: usize, n: usize| rule.goes_down(0x10_0000 + 4096 * 3 + distance, 0x10_0000, n);
+        for rule in [Rule::NEVER, Rule::AMD] {
+            assert!(!down(rule, 0, 8192), "{rule:?}: distance 0");
+            assert!(!down(rule, 16, 63), "{rule:?}: under 64 bytes");
+            assert!(!down(rule, 4080, 8192), "{rule:?}: 16 bytes below modulo 4 KiB");
+        }
+        assert!(!down(Rule::NEVER, 16, 8192));
+        assert!(down(Rule::AMD, 1, 64) && down(Rule::AMD, 31, 8192) && down(Rule::AMD, 16, 64 << 20));
+        assert!(!down(Rule::AMD, 32, 8192));
+        // A destination a page less 16 below its source is 16 above it modulo 4 KiB.
+        assert!(Rule::AMD.goes_down(0x10_0000, 0x10_0000 + 4080, 4080));
+    }
+}

kernel/src/arch/x86_64/boot.rs

@@ -4,7 +4,7 @@
 
 use toyos_abi::boot::{KernelArgs, MemoryMapEntry};
 
-use super::{apic, control_regs, idt, ioapic, microcode, pat, percpu};
+use super::{apic, control_regs, copy, cpu, idt, ioapic, microcode, pat, percpu};
 use crate::drivers::acpi::{self, MadtInfo};
 use crate::log;
 use crate::mm::Region;
@@ -64,6 +64,13 @@ pub fn after_console(args: &KernelArgs, _maps: &[MemoryMapEntry]) {
     microcode::init(args);
     microcode::load(0);
 
+    let rule = match cpu::vendor() {
+        kernel::cpu::Vendor::Amd => copy::Rule::AMD,
+        kernel::cpu::Vendor::Intel | kernel::cpu::Vendor::Other(_) => copy::Rule::NEVER,
+    };
+    copy::set_rule(rule);
+    log!("memcpy: highest first {rule:?}");
+
     // The read-back `pat::init` owes, on a boot that now has three channels to
     // carry a refusal.
     pat::check();

kernel/src/arch/x86_64/copy.rs

@@ -0,0 +1,156 @@
+//! The kernel's `memcpy`, in place of `compiler_builtins`' weak one: one
+//! `rep movsb`, or a descending copy where a forward one would stall.
+//!
+//! A load matching an older in-flight store on address bits 11:0 alone waits
+//! on it: AMD's Family 17h judges store-to-load forwarding by those bits (56305
+//! rev. 3.02, §2.12), and Intel calls it 4K aliasing (Intel 64 and IA-32
+//! Architectures Optimization Reference Manual). A forward copy whose
+//! destination is `d` bytes above its source modulo 4 KiB, `0 < d < n`, loads
+//! each byte `d` bytes after storing one with the same bits; copied highest
+//! first, the nearest older store with those bits is 4 KiB minus `d` back.
+//! The kernel touches no vector register (`build/fpgate.rs`), so the
+//! descending copy moves 8 bytes a store.
+//!
+//! The file uses nothing but `core`, so `kernel-copies` tests it on the host;
+//! only the freestanding kernel exports the symbol.
+
+#[cfg(target_os = "none")]
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+/// Which copies go highest first.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Rule {
+    /// Distances below this, modulo 4 KiB, go down.
+    pub window: usize,
+}
+
+/// The shortest copy that ever goes down: below it the stall is a few loads.
+const LEAST: usize = 64;
+
+impl Rule {
+    /// Intel's and an unmeasured vendor's: on Tiger Lake [`copy_down`] is
+    /// slower than `rep movsb` at every distance and length measured.
+    pub const NEVER: Self = Self { window: 0 };
+
+    /// Zen 2's: there, `rep movsb` waits at every distance under 32 bytes,
+    /// at every length measured from 4 to 64 KiB, and at none from 32 up.
+    pub const AMD: Self = Self { window: 32 };
+
+    /// Whether a copy of `n` bytes from `src` to `dest` goes down.
+    pub fn goes_down(self, dest: usize, src: usize, n: usize) -> bool {
+        let d = dest.wrapping_sub(src) % 4096;
+        n >= LEAST && d != 0 && d < self.window
+    }
+}
+
+/// The rule `memcpy` copies by, [`Rule::NEVER`] until [`set_rule`].
+#[cfg(target_os = "none")]
+static WINDOW: AtomicUsize = AtomicUsize::new(0);
+
+/// Set the rule every later copy goes by, before another CPU runs.
+#[cfg(target_os = "none")]
+pub fn set_rule(rule: Rule) {
+    WINDOW.store(rule.window, Ordering::Relaxed);
+}
+
+#[cfg(target_os = "none")]
+fn rule() -> Rule {
+    Rule { window: WINDOW.load(Ordering::Relaxed) }
+}
+
+/// Copy `n` bytes from `src` to `dest` as `rule` says.
+///
+/// # Safety
+/// `src` is readable and `dest` writable for `n` bytes, and the two ranges
+/// are disjoint or identical, as LLVM's `memcpy` allows.
+#[inline(always)]
+pub unsafe fn copy_by(dest: *mut u8, src: *const u8, n: usize, rule: Rule) {
+    // SAFETY: as this function's caller promises; `goes_down` holds `n` to at least 8.
+    unsafe {
+        if rule.goes_down(dest as usize, src as usize, n) {
+            copy_down(dest, src, n);
+        } else {
+            movsb(dest, src, n);
+        }
+    }
+}
+
+#[cfg(target_os = "none")]
+#[unsafe(no_mangle)]
+unsafe extern "C" fn memcpy(dest: *mut core::ffi::c_void, src: *const core::ffi::c_void, n: usize) -> *mut core::ffi::c_void {
+    // SAFETY: `memcpy`'s contract is `copy_by`'s.
+    unsafe { copy_by(dest.cast(), src.cast(), n, rule()) };
+    dest
+}
+
+/// Copy `n` bytes, lowest first.
+///
+/// # Safety
+/// As [`copy_by`].
+#[inline(always)]
+pub unsafe fn movsb(dest: *mut u8, src: *const u8, n: usize) {
+    // SAFETY: as this function's caller promises; every Ring 0 entry clears DF.
+    unsafe {
+        core::arch::asm!(
+            "rep movsb",
+            inout("rdi") dest => _,
+            inout("rsi") src => _,
+            inout("rcx") n => _,
+            options(nostack, preserves_flags),
+        );
+    }
+}
+
+/// Copy `n` bytes, at least 8, between disjoint ranges, highest first: the
+/// top 8 bytes in one store, then down 32 and 8 bytes a step from the
+/// destination's last 8-byte boundary, so no other store splits a line, and
+/// the bottom 8 bytes last. Stores past the boundary overlap one another,
+/// with the same bytes.
+///
+/// # Safety
+/// As [`copy_by`], with `n` at least 8.
+pub unsafe fn copy_down(dest: *mut u8, src: *const u8, n: usize) {
+    let aligned_end = ((dest as usize).wrapping_add(n) & !7).wrapping_sub(dest as usize);
+    // SAFETY: every access is inside `[0, n)` of its range, as the caller promises.
+    unsafe {
+        core::arch::asm!(
+            "mov {a}, [{s} + {n} - 8]",
+            "mov [{d} + {n} - 8], {a}",
+            "cmp {p}, 32",
+            "jb 3f",
+            "2:",
+            "sub {p}, 32",
+            "mov {a}, [{s} + {p} + 24]",
+            "mov {b}, [{s} + {p} + 16]",
+            "mov {c}, [{s} + {p} + 8]",
+            "mov {e}, [{s} + {p}]",
+            "mov [{d} + {p} + 24], {a}",
+            "mov [{d} + {p} + 16], {b}",
+            "mov [{d} + {p} + 8], {c}",
+            "mov [{d} + {p}], {e}",
+            "cmp {p}, 32",
+            "jae 2b",
+            "3:",
+            "cmp {p}, 8",
+            "jb 5f",
+            "4:",
+            "sub {p}, 8",
+            "mov {a}, [{s} + {p}]",
+            "mov [{d} + {p}], {a}",
+            "cmp {p}, 8",
+            "jae 4b",
+            "5:",
+            "mov {a}, [{s}]",
+            "mov [{d}], {a}",
+            s = in(reg) src,
+            d = in(reg) dest,
+            n = in(reg) n,
+            p = inout(reg) aligned_end => _,
+            a = out(reg) _,
+            b = out(reg) _,
+            c = out(reg) _,
+            e = out(reg) _,
+            options(nostack),
+        );
+    }
+}

kernel/src/arch/x86_64/mod.rs

@@ -29,6 +29,7 @@ pub mod boot;
 pub mod cache;
 pub mod clocksource;
 pub mod control_regs;
+pub mod copy;
 pub mod counters;
 pub mod console_uart;
 pub mod cpu;