diff --git a/Cargo.toml b/Cargo.toml index 611ac8a1..f8aebd0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ description = "rustc's frontend with no LLVM and no std: parsing through MIR, as repository = "https://github.com/pathscale/frontend" readme = "README.md" publish = true +# Measurement tools for working on this crate, not part of it. +exclude = ["tools/"] [lib] path = "src/lib.rs" diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..6467a113 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,51 @@ +# Tools for working on frontend + +Measurement and reproduction tools, used to find where the time goes. None of them is part of +the crate, and `cargo publish` leaves this directory out. + +**The rules still apply.** POSIX `sh` and bun only: no Python, perl, awk or node. Nothing here +runs in the build. + +## `bounded.sh` + +`bounded.sh SECONDS CMD...` runs a command and kills it after SECONDS, since macOS has no +`timeout`. Every tool below is run under it. It runs the command in the background, where stdin +is `/dev/null`, so to feed stdin wrap the command in `sh -c "... < file"`. + +## `profile/`: macOS `sample` profiles, folded with bun + +Take a profile of a running `parallel_timing` with `sample PID SECONDS -mayDie -f out.txt`, then: + +| Script | What it prints | +| --- | --- | +| `fold.ts FILE [filter]` | Inclusive and self samples per function | +| `kids.ts FILE NAME` | The direct callees of every frame whose name contains NAME | +| `parents.ts FILE NAME [levels]` | Caller chains of those frames | +| `subself.ts FILE... ROOT` | Self samples of the frames under ROOT, summed over files | +| `phase.ts FILE...` | The main thread's own time, charged to the innermost compiler pass on its stack. `PHASES="borrow check;type check bodies"` adds each pass's top functions | +| `share.ts FILE...` | The main thread's own time by kind: allocator, query machinery, hash tables, memory copies, interning, obligation processing | +| `agg.ts FILE` | Minimum and median of `LABEL ... check N ms` lines, per label, for A/B runs | +| `map.ts A B` | Per-pass seconds from two `-Z time-passes` JSON runs, side by side | + +`name.ts` decodes v0-mangled symbols for the others. + +## `parse-ref/`: frontend's parser against rust-analyzer's and syn's + +A crate of its own (not in the workspace). It times the three parsers over the same sources: +`cargo run --release --manifest-path tools/parse-ref/Cargo.toml -- DIR [ROUNDS]` (3 rounds by default; `ONLY_FRONTEND=1` times frontend alone). + +## `deps/`: reading a crate with its dependencies loaded, from source + +- **`chain.sh`:** reads a dependency chain with `frontend-facts`, in dependency order, each + crate once, `--items`, and writes each crate's metadata for the crates after it. It is the + exact recipe (`--cfg`, `--env`, `--extern`, `--standard-library`, `--proc-macro`) for std's + closure and for anyhow, serde_json and tokio with their closures. + - `chain.sh std` reads std's closure; `chain.sh reg` reads the three registry crates (run + `std` first). `chain.sh anyhow`, `serde_json` or `tokio` reads one. + - Set `LIBRARY` (a rust-src `library/` of frontend's upstream era, with its `vendor/`), + `REGISTRY` (a cargo registry source directory holding the named crate versions), `OUT` (the + output directory) and optionally `FACTS` (the `frontend-facts` binary) and `ARCH`. +- **`check.sh ok e0599 e0061`:** checks the fixtures beside it against the tokio chain. + - `ok.rs` checks clean. + - `e0599.rs` calls a method no type has (E0599). + - `e0061.rs` passes a wrong argument count (E0061). diff --git a/tools/bounded.sh b/tools/bounded.sh new file mode 100755 index 00000000..39329fec --- /dev/null +++ b/tools/bounded.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# bounded.sh SECONDS CMD...: run CMD, kill it after SECONDS. Output to stdout. +limit=$1; shift +"$@" & +p=$! +i=0 +while kill -0 $p 2>/dev/null; do + if [ $i -ge $limit ]; then kill -9 $p; echo "WATCHDOG-KILLED after ${limit}s: $*"; exit 124; fi + sleep 1; i=$((i+1)) +done +wait $p diff --git a/tools/deps/chain.sh b/tools/deps/chain.sh new file mode 100755 index 00000000..0540fa29 --- /dev/null +++ b/tools/deps/chain.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# chain.sh [std|reg|anyhow|serde_json|tokio]: read a dependency chain from source with +# frontend-facts, each crate once, in dependency order, writing each crate's metadata for the +# crates after it. `std` reads std's closure; `reg` reads the three registry crates' closures +# (run `std` first). Every crate is read `--items`. +# +# LIBRARY a rust-src `library/` of frontend's upstream era, with its `vendor/` +# REGISTRY a cargo registry source directory holding the registry crates named below +# OUT where metadata (`meta/lib.rmeta`), facts and logs go +# FACTS the `frontend-facts` binary (default `target/release/frontend-facts`) +# ARCH the target architecture std's build script would report (default `aarch64`) +set -u +here=$(cd "$(dirname "$0")" && pwd) +B=${FACTS:-target/release/frontend-facts} +L=${LIBRARY:?set LIBRARY to a rust-src library/ directory} +V=$L/vendor +R=${REGISTRY:?set REGISTRY to a cargo registry src directory} +S=${OUT:?set OUT to an output directory} +M=$S/meta +mkdir -p $M $S/out +CB="--extern noprelude:compiler_builtins=$M/libcompiler_builtins.rmeta" +# rd NAME ROOT EDITION ARGS... +rd() { + name=$1; root=$2; ed=$3; shift 3 + $here/../bounded.sh 600 /usr/bin/time -p $B --root $root --edition $ed --items ${LIBFLAG:-} "$@" --emit-metadata $M/lib$name.rmeta $name > $S/out/$name.json 2> $S/out/$name.err + rc=$? + t=$(grep '^real' $S/out/$name.err | tail -1) + n=$(grep -c '^error' $S/out/$name.err) + echo "$name rc=$rc $t errors=$n" + if [ $rc -ne 0 ]; then grep '^error' -A2 $S/out/$name.err | head -12; exit 1; fi +} +case "${1:-std}" in +std) +LIBFLAG=--standard-library +rd core $L/core/src/lib.rs 2024 +rd compiler_builtins $L/compiler-builtins/compiler-builtins/src/lib.rs 2024 --cfg 'feature="compiler-builtins"' --cfg 'feature="unmangled-names"' --extern core=$M/libcore.rmeta +rd alloc $L/alloc/src/lib.rs 2024 --extern core=$M/libcore.rmeta $CB +rd cfg_if $V/cfg-if-1.0.4/src/lib.rs 2018 --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="core"' --extern core=$M/libcore.rmeta $CB +rd libc $V/libc-0.2.189/src/lib.rs 2021 --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="align"' --cfg 'feature="rustc-std-workspace-core"' --extern rustc_std_workspace_core=$M/libcore.rmeta $CB +rd rustc_demangle $V/rustc-demangle-0.1.28/src/lib.rs 2015 --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="core"' --extern core=$M/libcore.rmeta $CB +rd hashbrown $V/hashbrown-0.17.1/src/lib.rs 2024 --cfg 'feature="rustc-dep-of-std"' --cfg 'feature="nightly"' --cfg 'feature="core"' --cfg 'feature="alloc"' --cfg 'feature="rustc-internal-api"' --extern core=$M/libcore.rmeta --extern alloc=$M/liballoc.rmeta $CB +rd unwind $L/unwind/src/lib.rs 2024 --extern core=$M/libcore.rmeta --extern libc=$M/liblibc.rmeta --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta $CB +rd panic_abort $L/panic_abort/src/lib.rs 2024 --extern core=$M/libcore.rmeta --extern alloc=$M/liballoc.rmeta $CB +rd std_detect $L/std_detect/src/lib.rs 2024 --extern core=$M/libcore.rmeta --extern alloc=$M/liballoc.rmeta --extern libc=$M/liblibc.rmeta --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta $CB +rd panic_unwind $L/panic_unwind/src/lib.rs 2024 --extern core=$M/libcore.rmeta --extern alloc=$M/liballoc.rmeta --extern libc=$M/liblibc.rmeta --extern unwind=$M/libunwind.rmeta --extern cfg_if=$M/libcfg_if.rmeta --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta $CB +rd std $L/std/src/lib.rs 2024 --cfg backtrace_in_libstd --env STD_ENV_ARCH=${ARCH:-aarch64} \ + --extern core=$M/libcore.rmeta --extern alloc=$M/liballoc.rmeta --extern libc=$M/liblibc.rmeta \ + --extern cfg_if=$M/libcfg_if.rmeta --extern hashbrown=$M/libhashbrown.rmeta --extern unwind=$M/libunwind.rmeta \ + --extern panic_abort=$M/libpanic_abort.rmeta --extern panic_unwind=$M/libpanic_unwind.rmeta --cfg 'feature="panic-unwind"' --extern std_detect=$M/libstd_detect.rmeta \ + --extern rustc_demangle=$M/librustc_demangle.rmeta \ + --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta $CB +LIBFLAG= +;; +esac +# std and everything std loads, for a crate that uses std. +STD="--extern std=$M/libstd.rmeta --extern noprelude:core=$M/libcore.rmeta --extern noprelude:alloc=$M/liballoc.rmeta $CB" +for d in libc cfg_if hashbrown unwind panic_abort panic_unwind std_detect rustc_demangle; do + STD="$STD --extern noprelude:$d=$M/lib$d.rmeta" +done +STD="$STD --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta" +case "${1:-std}" in +reg|anyhow) +rd anyhow $R/anyhow-1.0.104/src/lib.rs 2021 --cfg 'feature="std"' --cfg 'feature="default"' $STD +;; +esac +case "${1:-std}" in +reg|serde_json) +# serde_core `include!`s a file its build script writes; frontend runs no build script, so it is written here. +SERDE_OUT=$S/out-serde_core; mkdir -p $SERDE_OUT +printf '#[doc(hidden)]\npub mod __private229 {\n #[doc(hidden)]\n pub use crate::private::*;\n}\n' > $SERDE_OUT/private.rs +rd serde_core $R/serde_core-1.0.229/src/lib.rs 2021 --cfg 'feature="std"' --cfg 'feature="result"' --cfg 'feature="default"' --env OUT_DIR=$SERDE_OUT --env CARGO_PKG_VERSION_PATCH=229 $STD +rd itoa $R/itoa-1.0.18/src/lib.rs 2021 $STD +rd memchr $R/memchr-2.8.3/src/lib.rs 2021 --cfg 'feature="std"' --cfg 'feature="alloc"' $STD +rd zmij $R/zmij-1.0.23/src/lib.rs 2021 $STD +rd serde_json $R/serde_json-1.0.151/src/lib.rs 2021 --cfg 'feature="std"' --cfg 'feature="default"' --cfg 'fast_arithmetic="64"' \ + --extern serde_core=$M/libserde_core.rmeta --extern itoa=$M/libitoa.rmeta --extern memchr=$M/libmemchr.rmeta --extern zmij=$M/libzmij.rmeta $STD +;; +esac +case "${1:-std}" in +reg|tokio) +LIBFLAG=--standard-library; rd rustc_literal_escaper $V/rustc-literal-escaper-0.0.8/src/lib.rs 2021 --cfg 'feature="rustc-dep-of-std"' --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta --extern noprelude:rustc_std_workspace_std=$M/libstd.rmeta $STD; LIBFLAG= +LIBFLAG=--standard-library; rd proc_macro $L/proc_macro/src/lib.rs 2024 --extern rustc_literal_escaper=$M/librustc_literal_escaper.rmeta $STD; LIBFLAG= +PM="--extern proc_macro=$M/libproc_macro.rmeta --extern noprelude:rustc_literal_escaper=$M/librustc_literal_escaper.rmeta" +rd unicode_ident $R/unicode-ident-1.0.26/src/lib.rs 2021 $STD +rd proc_macro2 $R/proc-macro2-1.0.107/src/lib.rs 2021 --cfg 'feature="proc-macro"' --cfg 'feature="default"' --cfg wrap_proc_macro --extern unicode_ident=$M/libunicode_ident.rmeta $PM $STD +rd quote $R/quote-1.0.47/src/lib.rs 2021 --cfg 'feature="proc-macro"' --cfg 'feature="default"' --extern proc_macro2=$M/libproc_macro2.rmeta --extern noprelude:unicode_ident=$M/libunicode_ident.rmeta $PM $STD +SYNF=""; for f in derive parsing printing clone-impls proc-macro full default; do SYNF="$SYNF --cfg feature=\"$f\""; done +rd syn $R/syn-3.0.6/src/lib.rs 2021 $SYNF --extern proc_macro2=$M/libproc_macro2.rmeta --extern quote=$M/libquote.rmeta --extern unicode_ident=$M/libunicode_ident.rmeta $PM $STD +rd tokio_macros $R/tokio-macros-2.7.2/src/lib.rs 2021 --proc-macro --extern proc_macro2=$M/libproc_macro2.rmeta --extern quote=$M/libquote.rmeta --extern syn=$M/libsyn.rmeta --extern noprelude:unicode_ident=$M/libunicode_ident.rmeta $PM $STD +rd pin_project_lite $R/pin-project-lite-0.2.17/src/lib.rs 2018 $STD +TOKF=""; for f in sync macros rt; do TOKF="$TOKF --cfg feature=\"$f\""; done +rd tokio $R/tokio-1.53.1/src/lib.rs 2021 $TOKF --extern pin_project_lite=$M/libpin_project_lite.rmeta --extern tokio_macros=$M/libtokio_macros.rmeta --extern noprelude:proc_macro=$M/libproc_macro.rmeta $STD +;; +esac diff --git a/tools/deps/check.sh b/tools/deps/check.sh new file mode 100755 index 00000000..59dce3a0 --- /dev/null +++ b/tools/deps/check.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# check.sh FIXTURE...: check each fixture (`ok`, `e0599`, `e0061`, beside this script) against +# tokio and its closure, read by `chain.sh std` and `chain.sh tokio` into `$OUT/meta`. The +# answers are frontend-facts' `Checked` JSON, with the wall clock of each check. +# +# OUT the directory chain.sh wrote +# FACTS the `frontend-facts` binary (default `target/release/frontend-facts`) +set -u +here=$(cd "$(dirname "$0")" && pwd) +B=${FACTS:-target/release/frontend-facts} +S=${OUT:?set OUT to the directory chain.sh wrote} +M=$S/meta +D="--extern tokio=$M/libtokio.rmeta --extern std=$M/libstd.rmeta" +for d in core alloc compiler_builtins libc cfg_if hashbrown unwind panic_abort panic_unwind std_detect rustc_demangle pin_project_lite tokio_macros proc_macro rustc_literal_escaper; do + D="$D --extern noprelude:$d=$M/lib$d.rmeta" +done +D="$D --extern noprelude:rustc_std_workspace_core=$M/libcore.rmeta" +mkdir -p $S/out +for f in "$@"; do + # Through `sh -c`: the watchdog runs the command in the background, where stdin is /dev/null. + "$here/../bounded.sh" 120 sh -c "exec /usr/bin/time -p $B --check --edition 2021 $D user < $here/$f.rs" > $S/out/check-$f.json 2> $S/out/check-$f.err + echo "== $f rc=$? $(grep '^real' $S/out/check-$f.err)" + head -c 900 $S/out/check-$f.json; echo +done diff --git a/tools/deps/e0061.rs b/tools/deps/e0061.rs new file mode 100644 index 00000000..aa864aa2 --- /dev/null +++ b/tools/deps/e0061.rs @@ -0,0 +1,11 @@ +use tokio::sync::Mutex; + +pub async fn bump(m: &Mutex) -> i32 { + let mut guard = m.lock().await; + *guard += 1; + *guard +} + +pub fn make() -> Mutex { + Mutex::new(0, 1) +} diff --git a/tools/deps/e0599.rs b/tools/deps/e0599.rs new file mode 100644 index 00000000..593562cd --- /dev/null +++ b/tools/deps/e0599.rs @@ -0,0 +1,11 @@ +use tokio::sync::Mutex; + +pub async fn bump(m: &Mutex) -> i32 { + let mut guard = m.lock_all().await; + *guard += 1; + *guard +} + +pub fn make() -> Mutex { + Mutex::new(0) +} diff --git a/tools/deps/ok.rs b/tools/deps/ok.rs new file mode 100644 index 00000000..3d638ead --- /dev/null +++ b/tools/deps/ok.rs @@ -0,0 +1,11 @@ +use tokio::sync::Mutex; + +pub async fn bump(m: &Mutex) -> i32 { + let mut guard = m.lock().await; + *guard += 1; + *guard +} + +pub fn make() -> Mutex { + Mutex::new(0) +} diff --git a/tools/parse-ref/Cargo.toml b/tools/parse-ref/Cargo.toml new file mode 100644 index 00000000..724115df --- /dev/null +++ b/tools/parse-ref/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "parse-ref" +version = "0.1.0" +edition = "2024" +rust-version = "1.97" +publish = false + +[dependencies] +frontend = { path = "../..", features = ["diagnostics"] } +ra_ap_syntax = "^0.0.349" +syn = { version = "2", features = ["full", "parsing"], default-features = false } +proc-macro2 = { version = "1", default-features = false } + +[profile.release] +debug = false + +[workspace] diff --git a/tools/parse-ref/src/main.rs b/tools/parse-ref/src/main.rs new file mode 100644 index 00000000..1b2d4aa9 --- /dev/null +++ b/tools/parse-ref/src/main.rs @@ -0,0 +1,85 @@ +//! Parse-only throughput of three Rust parsers over the same files: frontend's `parses`, +//! rust-analyzer's parser building a rowan tree (`ra_ap_syntax`), and `syn`. Wall clock, +//! MB/s, and allocations counted by this binary's global allocator. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use std::time::Instant; + +struct Counting; +static ALLOCS: AtomicU64 = AtomicU64::new(0); +static BYTES: AtomicU64 = AtomicU64::new(0); +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(l.size() as u64, Relaxed); + unsafe { System.alloc(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, n: usize) -> *mut u8 { + ALLOCS.fetch_add(1, Relaxed); + BYTES.fetch_add(n as u64, Relaxed); + unsafe { System.realloc(p, l, n) } + } +} +#[global_allocator] +static A: Counting = Counting; + +fn catcher(f: &mut dyn FnMut()) -> Result<(), frontend::unwind_janky::Payload> { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) +} + +fn files(dir: &str) -> Vec { + let mut paths = Vec::new(); + let mut stack = vec![std::path::PathBuf::from(dir)]; + while let Some(d) = stack.pop() { + for e in std::fs::read_dir(&d).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { stack.push(p) } else if p.extension().is_some_and(|x| x == "rs") { paths.push(p) } + } + } + paths.sort(); + paths.iter().map(|p| std::fs::read_to_string(p).unwrap()).collect() +} + +fn time(name: &str, files: &[String], rounds: usize, mut f: impl FnMut(&str) -> bool) { + let bytes: usize = files.iter().map(String::len).sum(); + // One warm-up round, not timed. + let failed = files.iter().filter(|s| !f(s)).count(); + let (a0, b0) = (ALLOCS.load(Relaxed), BYTES.load(Relaxed)); + let start = Instant::now(); + for _ in 0..rounds { + for s in files { + std::hint::black_box(f(s)); + } + } + let secs = start.elapsed().as_secs_f64() / rounds as f64; + let allocs = (ALLOCS.load(Relaxed) - a0) / rounds as u64; + let abytes = (BYTES.load(Relaxed) - b0) / rounds as u64; + println!( + "{name:<22} {:>8.1} ms {:>8.1} MB/s {:>11} allocs {:>7.1} allocs/KB {:>8.1} MB alloc ({failed} files refused)", + secs * 1e3, + bytes as f64 / 1e6 / secs, + allocs, + allocs as f64 / (bytes as f64 / 1e3), + abytes as f64 / 1e6, + ); +} + +fn main() { + frontend::unwind_janky::install_catcher(catcher); + std::panic::set_hook(Box::new(|_| {})); + let dir = std::env::args().nth(1).expect("a directory of .rs files"); + let rounds: usize = std::env::args().nth(2).and_then(|n| n.parse().ok()).unwrap_or(3); + let files = files(&dir); + let bytes: usize = files.iter().map(String::len).sum(); + println!("{} files, {:.1} MB, {rounds} rounds, single thread", files.len(), bytes as f64 / 1e6); + time("frontend parses", &files, rounds, |s| frontend::frontend_facts::syntax::parses(s).is_ok()); + if std::env::var_os("ONLY_FRONTEND").is_some() { return; } + time("rust-analyzer (rowan)", &files, rounds, |s| { + ra_ap_syntax::SourceFile::parse(s, ra_ap_syntax::Edition::Edition2024).errors().is_empty() + }); + time("syn", &files, rounds, |s| syn::parse_file(s).is_ok()); +} diff --git a/tools/profile/agg.ts b/tools/profile/agg.ts new file mode 100644 index 00000000..355340c5 --- /dev/null +++ b/tools/profile/agg.ts @@ -0,0 +1,4 @@ +const t = await Bun.file(Bun.argv[2]).text(); +const m = new Map(); +for (const l of t.split("\n")) { const x = l.match(/^(\S+) .*check ([\d.]+) ms/); if (x) { (m.get(x[1]) ?? m.set(x[1], []).get(x[1])!).push(+x[2]); } } +for (const [k, v] of m) { v.sort((a,b)=>a-b); console.log(k, "min", v[0], "median", v[Math.floor(v.length/2)], "all", v.join(" ")); } diff --git a/tools/profile/fold.ts b/tools/profile/fold.ts new file mode 100644 index 00000000..632c9f50 --- /dev/null +++ b/tools/profile/fold.ts @@ -0,0 +1,40 @@ +// fold.ts SAMPLE_FILE [filter]: inclusive and self sample counts per function from macOS `sample`. +const text = await Bun.file(Bun.argv[2]).text(); +const filter = Bun.argv[3] ?? ""; +const lines = text.split("\n"); +const start = lines.findIndex((l) => l.startsWith("Call graph:")); +const end = lines.findIndex((l, i) => i > start && l.startsWith("Total number in stack")); +import { idents } from "./name.ts"; +function name(sym: string): string { + const m = sym.match(/_R[A-Za-z0-9_]+/); + if (!m) return sym.replace(/\s+\(in .*$/, "").replace(/\s+\+ .*$/, "").trim().slice(0, 60); + const parts = idents(m[0]) + .filter((p) => !/^(frontend|std|core|alloc)$/.test(p)); + return parts.slice(-2).join("::"); +} +const incl = new Map(), self = new Map(); +const stack: { depth: number; n: number; name: string; kids: number }[] = []; +const flush = (upto: number) => { + while (stack.length && stack[stack.length - 1].depth >= upto) { + const f = stack.pop()!; + self.set(f.name, (self.get(f.name) ?? 0) + Math.max(0, f.n - f.kids)); + } +}; +for (const l of lines.slice(start + 1, end)) { + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); + if (!m) continue; + const depth = m[1].length, n = +m[2], nm = name(m[3]); + flush(depth); + if (stack.length) stack[stack.length - 1].kids += n; + // count inclusive once per distinct name on the current path + if (!stack.some((f) => f.name === nm)) incl.set(nm, (incl.get(nm) ?? 0) + n); + stack.push({ depth, n, name: nm, kids: 0 }); +} +flush(0); +const show = (title: string, m: Map) => { + console.log(`== ${title}`); + [...m].filter(([k]) => k.includes(filter)).sort((a, b) => b[1] - a[1]).slice(0, 25) + .forEach(([k, v]) => console.log(String(v).padStart(7), k)); +}; +show("self", self); +show("inclusive", incl); diff --git a/tools/profile/kids.ts b/tools/profile/kids.ts new file mode 100644 index 00000000..22357bef --- /dev/null +++ b/tools/profile/kids.ts @@ -0,0 +1,25 @@ +// kids.ts SAMPLE_FILE NAME_SUBSTRING: direct callees of every frame whose name contains NAME, summed. +import { idents } from "./name.ts"; +const [file, want] = [Bun.argv[2], Bun.argv[3]]; +const lines = (await Bun.file(file).text()).split("\n"); +function name(sym: string): string { + const m = sym.match(/_R[A-Za-z0-9_]+/); + if (!m) return sym.replace(/\s+\(in .*$/, "").replace(/\s+\+ .*$/, "").trim().slice(0, 60); + return idents(m[0]).filter((p) => !/^(frontend|std|core|alloc)$/.test(p)).slice(-2).join("::"); +} +const kids = new Map(); +let total = 0; +const stack: { depth: number; name: string; n: number }[] = []; +for (const l of lines) { + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); + if (!m) continue; + const depth = m[1].length, n = +m[2], nm = name(m[3]); + while (stack.length && stack[stack.length - 1].depth >= depth) stack.pop(); + const parent = stack[stack.length - 1]; + const inside = stack.some((f, i) => f.name.includes(want) && i < stack.length - 1); + if (parent && parent.name.includes(want) && !inside) { kids.set(nm, (kids.get(nm) ?? 0) + n); } + if (nm.includes(want) && !stack.some((f) => f.name.includes(want))) total += n; + stack.push({ depth, name: nm, n }); +} +console.log(`${want}: ${total}`); +[...kids].sort((a, b) => b[1] - a[1]).slice(0, 20).forEach(([k, v]) => console.log(String(v).padStart(7), k)); diff --git a/tools/profile/map.ts b/tools/profile/map.ts new file mode 100644 index 00000000..2f44ac13 --- /dev/null +++ b/tools/profile/map.ts @@ -0,0 +1,17 @@ +// map.ts A B: per-pass seconds summed over a run, side by side. +const read = async (f: string) => { + const m = new Map(); + for (const line of (await Bun.file(f).text()).split("\n")) { + if (!line.startsWith("time: {")) continue; + let e; try { e = JSON.parse(line.slice(6)); } catch { continue; } + m.set(e.pass, (m.get(e.pass) ?? 0) + e.time); + } + return m; +}; +const [a, b] = [await read(Bun.argv[2]), await read(Bun.argv[3])]; +const keys = [...new Set([...a.keys(), ...b.keys()])].sort((x, y) => (a.get(y) ?? 0) - (a.get(x) ?? 0)); +for (const k of keys.slice(0, 40)) { + const x = (a.get(k) ?? 0) * 1000, y = (b.get(k) ?? 0) * 1000; + if (x < 1 && y < 1) continue; + console.log(`${x.toFixed(0).padStart(7)} ${y.toFixed(0).padStart(7)} ${(x / (y || 1)).toFixed(2).padStart(6)}x ${k}`); +} diff --git a/tools/profile/name.ts b/tools/profile/name.ts new file mode 100644 index 00000000..e031d703 --- /dev/null +++ b/tools/profile/name.ts @@ -0,0 +1,15 @@ +export function idents(sym: string): string[] { + const out: string[] = []; + let i = 0; + while (i < sym.length) { + if (/[1-9]/.test(sym[i])) { + let j = i; + while (j < sym.length && /[0-9]/.test(sym[j])) j++; + const n = +sym.slice(i, j); + const id = sym.slice(j, j + n); + if (id.length === n && /^[A-Za-z_][A-Za-z0-9_]*$/.test(id)) { out.push(id); i = j + n; continue; } + } + i++; + } + return out; +} diff --git a/tools/profile/parents.ts b/tools/profile/parents.ts new file mode 100644 index 00000000..fc054cc0 --- /dev/null +++ b/tools/profile/parents.ts @@ -0,0 +1,23 @@ +// parents.ts SAMPLE_FILE NAME [levels]: caller chains of frames whose name contains NAME. +import { idents } from "./name.ts"; +const [file, want, lv] = [Bun.argv[2], Bun.argv[3], +(Bun.argv[4] ?? 8)]; +const lines = (await Bun.file(file).text()).split("\n"); +function name(sym: string): string { + const m = sym.match(/_R[A-Za-z0-9_]+/); + if (!m) return sym.replace(/\s+\(in .*$/, "").replace(/\s+\+ .*$/, "").trim().slice(0, 50); + return idents(m[0]).filter((p) => !/^(frontend|std|core|alloc)$/.test(p)).slice(-2).join("::"); +} +const chains = new Map(); +const stack: { depth: number; name: string }[] = []; +for (const l of lines) { + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); + if (!m) continue; + const depth = m[1].length, n = +m[2], nm = name(m[3]); + while (stack.length && stack[stack.length - 1].depth >= depth) stack.pop(); + if (nm.includes(want)) { + const chain = stack.slice(-lv).map((f) => f.name).filter((x) => !x.startsWith(" " : ""); + chains.set(chain, (chains.get(chain) ?? 0) + n); + } + stack.push({ depth, name: nm }); +} +[...chains].sort((a, b) => b[1] - a[1]).slice(0, 12).forEach(([k, v]) => console.log(String(v).padStart(6), k)); diff --git a/tools/profile/phase.ts b/tools/profile/phase.ts new file mode 100644 index 00000000..6217eb53 --- /dev/null +++ b/tools/profile/phase.ts @@ -0,0 +1,62 @@ +// phase.ts SAMPLE... : self samples of the main thread charged to the innermost pass on the stack. +import { idents } from "./name.ts"; +const specific: [string, string][] = [ + ["rustc_borrowck", "borrow check"], ["check_liveness", "liveness"], ["rustc_mir_build", "MIR build"], + ["rustc_mir_transform", "MIR passes"], ["rustc_mir_dataflow", "MIR passes"], ["rustc_hir_typeck", "type check bodies"], + ["wfcheck", "well-formedness"], ["coherence", "coherence"], ["rustc_lint", "lints"], ["rustc_privacy", "privacy"], + ["rustc_passes", "misc passes"], ["rustc_pattern_analysis", "match checking"], ["rustc_ast_lowering", "lowering"], + ["late_resolve_crate", "late resolution"], ["rustc_resolve", "resolution"], ["rustc_expand", "expansion"], + ["rustc_builtin_macros", "expansion"], ["rustc_parse", "parse"], ["rustc_lexer", "parse"], + ["rustc_ast_passes", "AST checks"], ["rustc_hir_analysis", "hir analysis (signatures, collect)"], + ["rustc_ty_utils", "hir analysis (signatures, collect)"], +]; +const general: [string, string][] = [ + ["drop_in_place", "teardown"], ["new_lint_store", "session setup"], ["create_global_ctxt", "session setup"], + ["run_compiler", "session setup/other"], +]; +const totals = new Map(); let all = 0; +const byFn = new Map>(); +for (const file of Bun.argv.slice(2)) { + const lines = (await Bun.file(file).text()).split("\n"); + let inMain = false; + const stack: { depth: number; n: number; kids: number; parts: string }[] = []; + const charge = (f: { n: number; kids: number }, path: string[]) => { + const self = Math.max(0, f.n - f.kids); if (!self) return; + let phase = ""; + for (const table of [specific, general]) { + for (let i = path.length - 1; i >= 0 && !phase; i--) { + const hit = table.find(([pat]) => path[i].includes(pat)); if (hit) phase = hit[1]; + } + if (phase) break; + } + phase ||= "other"; + totals.set(phase, (totals.get(phase) ?? 0) + self); all += self; + const leaf = path[path.length - 1].split("::").slice(-3).join("::"); + const per = byFn.get(phase) ?? byFn.set(phase, new Map()).get(phase)!; + per.set(leaf, (per.get(leaf) ?? 0) + self); + }; + const pop = (upto: number) => { while (stack.length && stack[stack.length-1].depth >= upto) { const f = stack.pop()!; charge(f, [...stack.map(s => s.parts), f.parts]); } }; + for (const l of lines) { + if (l.startsWith("Total number in stack")) break; + if (l.includes("Thread_")) { pop(0); inMain = l.includes("main-thread"); continue; } + if (!inMain) continue; + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); if (!m) continue; + const depth = m[1].length, n = +m[2]; + const sym = m[3]; const mm = sym.match(/_R[A-Za-z0-9_]+/); + const parts = mm ? idents(mm[0]).join("::") : sym.replace(/\s+\(in .*$/, ""); + pop(depth); + if (stack.length) stack[stack.length-1].kids += n; + stack.push({ depth, n, kids: 0, parts }); + } + pop(0); +} +const runs = Bun.argv.length - 2; +console.log(`main thread, ${runs} runs, ${(all/runs).toFixed(0)} samples per run (1 ms each)`); +[...totals].sort((a,b)=>b[1]-a[1]).forEach(([k,v]) => console.log(String(Math.round(v/runs)).padStart(6), (100*v/all).toFixed(1).padStart(5)+"%", k)); + +for (const want of (Bun.env.PHASES ?? "").split(";").filter(Boolean)) { + const per = byFn.get(want); if (!per) continue; + const tot = [...per.values()].reduce((a, b) => a + b, 0); + console.log("\n== " + want + " (" + Math.round(tot / runs) + " per run), own time by function"); + [...per].sort((a,b)=>b[1]-a[1]).slice(0, +(Bun.env.TOP ?? 18)).forEach(([k,v]) => console.log(String(Math.round(v/runs)).padStart(6), (100*v/tot).toFixed(1).padStart(5)+"%", k)); +} diff --git a/tools/profile/share.ts b/tools/profile/share.ts new file mode 100644 index 00000000..4c22b0ab --- /dev/null +++ b/tools/profile/share.ts @@ -0,0 +1,28 @@ +import { idents } from "./name.ts"; +const cats: [RegExp, string][] = [ + [/rustc_query_impl|query::plumbing|query::caches|try_execute_query|execute_job|JobOwner|QueryJob|query::job/, "query machinery"], + [/dep_graph/, "dep graph"], + [/^_xzm|malloc|_free$|free_|GlobalAlloc|alloc_zeroed|__rust_alloc|__rust_dealloc|realloc/, "allocator"], + [/memmove|memcpy|memset|bzero/, "memory copy/zero"], + [/intern_|Interned|InternSet|mk_args|mk_ty|mk_type_list|intern_predicate/, "interning"], + [/obligation_forest|fulfill|FulfillProcessor|process_obligations|register_obligation/, "obligation processing"], + [/FxHash|hashbrown|FxBuildHasher|RawTable/, "hash tables"], +]; +const tot = new Map(); let all = 0; +for (const file of Bun.argv.slice(2)) { + const lines = (await Bun.file(file).text()).split("\n"); let inMain = false; + const st: { d: number; n: number; k: number; name: string }[] = []; + const pop = (u: number) => { while (st.length && st[st.length-1].d >= u) { const f = st.pop()!; const s = Math.max(0, f.n - f.k); if (!s) continue; all += s; const c = cats.find(([re]) => re.test(f.name)); const key = c ? c[1] : "compiler logic"; tot.set(key, (tot.get(key) ?? 0) + s); } }; + for (const l of lines) { + if (l.startsWith("Total number in stack")) break; + if (l.includes("Thread_")) { pop(0); inMain = l.includes("main-thread"); continue; } + if (!inMain) continue; + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); if (!m) continue; + const d = m[1].length, n = +m[2]; const mm = m[3].match(/_R[A-Za-z0-9_]+/); + const name = mm ? idents(mm[0]).join("::") : m[3].replace(/\s+\(in .*$/, "").trim(); + pop(d); if (st.length) st[st.length-1].k += n; st.push({ d, n, k: 0, name }); + } + pop(0); +} +const runs = Bun.argv.length - 2; +[...tot].sort((a,b)=>b[1]-a[1]).forEach(([k,v]) => console.log(String(Math.round(v/runs)).padStart(6), (100*v/all).toFixed(1).padStart(5)+"%", k)); diff --git a/tools/profile/subself.ts b/tools/profile/subself.ts new file mode 100644 index 00000000..7ed55cfe --- /dev/null +++ b/tools/profile/subself.ts @@ -0,0 +1,27 @@ +// subself.ts SAMPLE... -- ROOT: self samples of frames under ROOT (by name substring), summed over files. +import { idents } from "./name.ts"; +const args = Bun.argv.slice(2); const root = args.pop()!; +function name(sym: string): string { + const m = sym.match(/_R[A-Za-z0-9_]+/); + if (!m) return sym.replace(/\s+\(in .*$/, "").replace(/\s+\+ .*$/, "").trim().slice(0, 60); + return idents(m[0]).filter((p) => !/^(frontend|std|core|alloc)$/.test(p)).slice(-3).join("::"); +} +const self = new Map(); let total = 0; +for (const file of args) { + const lines = (await Bun.file(file).text()).split("\n"); + const stack: { depth: number; n: number; name: string; kids: number; inRoot: boolean }[] = []; + const pop = (upto: number) => { while (stack.length && stack[stack.length-1].depth >= upto) { const f = stack.pop()!; if (f.inRoot) { const s = Math.max(0, f.n - f.kids); self.set(f.name, (self.get(f.name) ?? 0) + s); } } }; + for (const l of lines) { + const m = l.match(/^([ +!:|]*)(\d+) (.*)$/); if (!m) continue; + const depth = m[1].length, n = +m[2], nm = name(m[3]); + pop(depth); + const parentIn = stack.length ? stack[stack.length-1].inRoot : false; + if (stack.length) stack[stack.length-1].kids += n; + const isRoot = !parentIn && nm.includes(root); + if (isRoot) total += n; + stack.push({ depth, n, name: nm, kids: 0, inRoot: parentIn || isRoot }); + } + pop(0); +} +console.log(`${root}: ${total} samples`); +[...self].sort((a,b)=>b[1]-a[1]).slice(0, +(Bun.env.TOP ?? 30)).forEach(([k,v]) => console.log(String(v).padStart(6), (100*v/total).toFixed(1).padStart(5)+'%', k));