diff --git a/README.md b/README.md index 027a6b2a7d..191f91b659 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1984** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1985** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1984 contracts across inference, training, quantization, attention, FFN, +The tree carries 1985 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/contracts.nt b/contracts/contracts.nt index f0d8c3a042..68baa3779d 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -9716,6 +9716,13 @@ "course-flow-batch-wip-v1"^^ . "course-flow-batch-wip"^^ . "1.0.0"^^ . + . + . + . + "contracts/course-flow-content-factory-v2-v1.yaml"^^ . + "course-flow-content-factory-v2-v1"^^ . + "course-flow-content-factory-v2"^^ . + "1.0.0"^^ . . . . @@ -23389,6 +23396,7 @@ "crates/"^^ . "crates/apr-cli/"^^ . "crates/apr-cli/Cargo.toml"^^ . + "crates/aprender-compute"^^ . "crates/aprender-compute/"^^ . "crates/aprender-contracts-cli/"^^ . "crates/aprender-contracts-macros/"^^ . @@ -23413,7 +23421,6 @@ "crates/aprender-train/"^^ . "docs/BEATS.md"^^ . "docs/specifications/aprender-monorepo-consolidation.md"^^ . - "docs/ssc-training-status-2026-03-22.md"^^ . "scripts/apr_bin.sh"^^ . "scripts/bump-version.sh"^^ . "scripts/check_clippy_current_stable.sh"^^ . diff --git a/contracts/course-flow-content-factory-v2-v1.yaml b/contracts/course-flow-content-factory-v2-v1.yaml new file mode 100644 index 0000000000..da9dfaaf60 --- /dev/null +++ b/contracts/course-flow-content-factory-v2-v1.yaml @@ -0,0 +1,85 @@ +metadata: + version: "1.0.0" + created: "2026-10-18" + author: PAIML Engineering + registry: true + references: + - "Little, J.D.C. (1961), A Proof for the Queuing Formula L = lambda W." + - "Denning, P.J. and Buzen, J.P. (1978), The Operational Analysis of Queueing Network Models (utilization and forced flow laws)." + - "flow model spec, internal (eqs 1, 2, 3, 5, 6, 14)" + description: "course_flow Content Factory v2: fan-out to three renditions, rework loop, GPU priority, build work vs elapsed" + +name: course-flow-content-factory-v2 +version: "1.0.0" +scope: "crates/aprender-simulate/src/scenarios/course_flow.rs" + +description: | + Content Factory v2 is opt-in on CourseFlowConfig; the defaults reproduce the + original course_flow model. With fan_out, every master fans out to R = 3 + renditions with no split, so every course takes the staging lanes and the + stage busy fraction meets the utilization law. A rework loop sends a share + alpha of masters back to build, so build carries B / (1 - alpha) passes a day. + The GPU serves the trunk first while the recent course W exceeds a threshold, + else Shorts, and carries LinkedIn feed renders beside the Shorts. Build work + time (holds a builder; sizing) is split from build elapsed time (lead time). + +equations: + fan_out_forced_flow: + description: "launches per day = B * R with R = 3 (flow model eq 3)" + formula: "launches_per_day = 3 * courses_per_day" + + stage_utilization: + description: "stage busy = B * t_sc / (1440 * lanes) = 5 * 360 / (1440 * 2) = 0.625 (flow model eq 6)" + formula: "|stage_util - B * t_sc / (1440 * lanes)| <= 0.02" + + rework_build_utilization: + description: "build busy = B * t_b / (1440 * N_b) / (1 - alpha); at alpha = 0.16 that is 0.795 / 0.84 <= 0.95 (flow model eqs 2, 14)" + formula: "util_build <= 0.95 and |util_build - 0.795 / (1 - alpha)| <= 0.02" + + gpu_utilization: + description: "GPU busy = (B * t_g + (B * k + 4) * t_v) / 1440 (flow model eq 5)" + formula: "|util_render - (B * t_g + (B * k + 4) * t_v) / 1440| <= 0.02" + + gpu_priority_rule: + description: "trunk first while the recent course W > threshold, else Shorts; the rule reorders and cannot add GPU minutes" + formula: "pick(render) = course iff W_recent > w_threshold" + + build_work_vs_elapsed: + description: "a builder holds t_b_work (sizing); lead time carries t_b_elapsed (flow model eqs 1, 2)" + formula: "util_build = B * t_b_work / (1440 * N_b) and dW = t_b_elapsed - t_b_work" + +falsification: + - id: FALSIFY-COURSE_FLOW_CF_V2_001 + description: "the stage busy fraction matches the eq 6 law 62.5% within 2 points under fan-out" + test: "coursera_stage_matches_the_law_within_two_points in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_002 + description: "at alpha = 0.16 build busy is at most 95% and within 2 points of 0.795 / (1 - alpha)" + test: "rework_at_alpha_016_keeps_build_at_or_under_95_percent in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_003 + description: "fan-out launches three renditions per master; the legacy split launches one" + test: "fan_out_launches_three_renditions_per_master in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_004 + description: "the GPU serves the trunk first only while W is over the threshold, and only on the GPU" + test: "gpu_serves_trunk_first_only_while_w_is_over_the_threshold in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_005 + description: "GPU busy matches the eq 5 law with Shorts and four LinkedIn feed renders a day" + test: "gpu_carries_trunk_shorts_and_four_linkedin_renders in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_006 + description: "build work sizes the station; build elapsed beyond work moves lead time, not build busy" + test: "build_work_sizes_the_station_and_elapsed_drives_lead_time in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" + + - id: FALSIFY-COURSE_FLOW_CF_V2_007 + description: "the v2 inputs round-trip through the ledger inputs and the sim block" + test: "v2_inputs_round_trip_through_the_ledger_and_sim_block in crates/aprender-simulate/src/scenarios/course_flow.rs" + evidence: "cargo test -p aprender-simulate --lib course_flow" diff --git a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json index 6bb276cb32..e38191390b 100644 --- a/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json +++ b/contracts/witness/811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "7597b4ad189f534dd58cda55c9b1fe29e6339ddb77fd5f40969b4a028eed8f88", + "census_id_set_sha256": "1ff4f1bffcb59d87af86465a6fb83b5959988801194fa4436da56a73b2c62b8e", "relations_sha256": "811c78184eb2609244c59534f015f0be2f90923120a3027c176ad56723d0d850", - "reasoner_git_sha": "461bf9ef1921a8fbea4bc338f21b566ec342a95e", + "reasoner_git_sha": "4c0a87672d499188eb59bc4ea6969dfdc8beff45", "checkable_n": 87, "result": { "kind": "model", diff --git a/crates/aprender-simulate/src/scenarios/course_flow.rs b/crates/aprender-simulate/src/scenarios/course_flow.rs index 3c5dbe3d41..fc46be46b1 100644 --- a/crates/aprender-simulate/src/scenarios/course_flow.rs +++ b/crates/aprender-simulate/src/scenarios/course_flow.rs @@ -4,6 +4,12 @@ //! launch with no human step; Coursera courses wait for the daily 07:00 UTC submission batch. //! Shorts take priority over courses on the shared stations (render GPU, Chrome lane). Service //! times are Erlang-4 (gamma, coefficient of variation 0.5). Time is minutes from 00:00 UTC. +//! +//! Content Factory v2 (flow model B5) is opt-in through the config: every master fans out to +//! R = 3 renditions with no split (eq 3), a rework loop sends a share alpha of masters back to +//! build (eq 14), the GPU serves the trunk first while the recent course W exceeds a threshold +//! (eq 5), LinkedIn feed renders share the GPU, and build work time (eq 2, sizing) is split +//! from build elapsed time (eq 1, lead time). The defaults reproduce the original model. use std::collections::VecDeque; @@ -20,11 +26,16 @@ const RENDER: usize = 3; const VERIFY: usize = 4; const CHROME: usize = 5; const NAMES: [&str; 6] = ["outline", "build", "design", "render", "verify", "chrome"]; +/// Renditions per master under fan-out (flow model eq 3): edX, Coursera, LinkedIn Learning. +const FAN_OUT_R: f64 = 3.0; +/// Completed courses the GPU priority rule averages to estimate the current W. +const W_WINDOW: usize = 20; /// Inputs of the factory. Times are minutes per course; `*_lanes`/`builders`/`gpus` are servers. #[derive(Debug, Clone, PartialEq)] pub struct CourseFlowConfig { pub builders: usize, + /// Builder-held work per build pass (flow model eq 2, sizing). pub build_min: f64, pub design_lanes: usize, pub design_min: f64, @@ -40,6 +51,19 @@ pub struct CourseFlowConfig { pub verify_min: f64, /// Count a finished course as WIP until the daily batch releases it (flow model B6). pub batch_in_wip: bool, + /// Every master fans out to all R = 3 renditions, so every course takes the stage (eq 3). + pub fan_out: bool, + /// Share of masters the distinctness gate sends back to build (eq 14); 0 disables the loop. + pub rework_alpha: f64, + /// GPU serves the trunk first while the recent course W in days exceeds this; `None` keeps + /// Shorts first always. + pub trunk_first_w_days: Option, + /// LinkedIn feed renders per day on the GPU (eq 5's "+ 4"). + pub linkedin_renders_per_day: f64, + /// GPU minutes per vertical or feed render (eq 5's t_v). + pub vertical_render_min: f64, + /// Wall-clock minutes from build start to build done (eq 1); below `build_min` it equals it. + pub build_elapsed_min: f64, } /// What one seeded run measured after the warm-up. @@ -52,6 +76,16 @@ pub struct CourseFlowResult { pub util: [f64; 6], /// Mean hours a built course waits for the daily batch (courses that take the batch path). pub mean_batch_wait_h: f64, + /// Busy fraction of the stage lanes from course staging alone (eq 6). + pub stage_util: f64, + /// Platform launches per day: R per master under fan-out, one per course otherwise (eq 3). + pub launches_per_day: f64, + /// LinkedIn feed renders finished per day. + pub feed_renders_per_day: f64, + /// Mean builder-held hours per build pass (eq 2, sizing). + pub t_b_work_h: f64, + /// Mean hours from entering the build queue to leaving build, per pass (eq 1, lead time). + pub t_b_elapsed_h: f64, } impl CourseFlowResult { @@ -73,6 +107,7 @@ struct Job { enum Ev { Start, ShortTick, + FeedTick, Done(usize, Job), Delay(Job), Batch, @@ -82,6 +117,7 @@ fn pack(e: &Ev) -> SimEvent { let (name, st, j) = match e { Ev::Start => ("start", 0, None), Ev::ShortTick => ("short_tick", 0, None), + Ev::FeedTick => ("feed_tick", 0, None), Ev::Done(s, j) => ("done", *s, Some(*j)), Ev::Delay(j) => ("delay", 0, Some(*j)), Ev::Batch => ("batch", 0, None), @@ -108,6 +144,7 @@ fn unpack(e: &SimEvent) -> Option { Some(match name.as_str() { "start" => Ev::Start, "short_tick" => Ev::ShortTick, + "feed_tick" => Ev::FeedTick, "done" => Ev::Done((v >> 56) as usize, job), "delay" => Ev::Delay(job), "batch" => Ev::Batch, @@ -123,6 +160,17 @@ struct Station { busy_min: f64, } +/// Post-warm-up sums the v2 result fields are built from. +#[derive(Default)] +struct V2Tally { + stage_busy: f64, + build_work: f64, + build_work_n: u64, + build_elapsed: f64, + build_elapsed_n: u64, + feeds_done: u64, +} + struct Sim { c: CourseFlowConfig, rng: SimRng, @@ -134,17 +182,56 @@ struct Sim { pending: usize, started_at: Vec, coursera: Vec, + build_in: Vec, + recent_w: VecDeque, batch_wait: Vec<(usize, f64)>, bw_sum: f64, bw_n: u64, sojourn: Vec, shorts_done: u64, + v2: V2Tally, in_system: usize, l_area: f64, last_t: f64, } impl Sim { + fn new(c: &CourseFlowConfig, seed: u64, warmup_d: f64) -> Self { + let size = [1, c.builders, c.design_lanes, c.gpus, 4, c.chrome_lanes]; + Self { + c: c.clone(), + rng: SimRng::new(seed), + sched: EventScheduler::new(), + now: 0.0, + warm: warmup_d * DAY, + st: size + .iter() + .map(|&n| Station { + free: n, + size: n, + hi: VecDeque::new(), + lo: VecDeque::new(), + busy_min: 0.0, + }) + .collect(), + wip: 0, + pending: 0, + started_at: Vec::new(), + coursera: Vec::new(), + build_in: Vec::new(), + recent_w: VecDeque::new(), + batch_wait: Vec::new(), + bw_sum: 0.0, + bw_n: 0, + sojourn: Vec::new(), + shorts_done: 0, + v2: V2Tally::default(), + in_system: 0, + l_area: 0.0, + last_t: 0.0, + } + } + fn at(&mut self, dt: f64, e: &Ev) { self.sched .schedule(SimTime::from_secs((self.now + dt) * 60.0), pack(e)); @@ -165,18 +252,44 @@ impl Sim { self.dispatch(s); } + /// The GPU priority rule: trunk first while the recent course W exceeds the threshold. + fn trunk_first(&self) -> bool { + let Some(limit) = self.c.trunk_first_w_days else { + return false; + }; + let n = self.recent_w.len(); + n > 0 && self.recent_w.iter().sum::() / n as f64 / DAY > limit + } + + fn next_job(&mut self, s: usize) -> Option<(Job, f64)> { + let trunk = s == RENDER && self.trunk_first(); + let st = &mut self.st[s]; + if trunk { + st.lo.pop_front().or_else(|| st.hi.pop_front()) + } else { + st.hi.pop_front().or_else(|| st.lo.pop_front()) + } + } + + fn account(&mut self, s: usize, job: Job, d: f64) { + self.st[s].busy_min += d; + if s == CHROME && !job.short { + self.v2.stage_busy += d; + } + if s == BUILD { + self.v2.build_work += d; + self.v2.build_work_n += 1; + } + } + fn dispatch(&mut self, s: usize) { while self.st[s].free > 0 { - let Some((job, d)) = self.st[s] - .hi - .pop_front() - .or_else(|| self.st[s].lo.pop_front()) - else { + let Some((job, d)) = self.next_job(s) else { break; }; self.st[s].free -= 1; if self.now >= self.warm { - self.st[s].busy_min += d; + self.account(s, job, d); } self.at(d, &Ev::Done(s, job)); } @@ -185,7 +298,8 @@ impl Sim { fn start_course(&mut self) { let id = self.started_at.len(); self.started_at.push(self.now); - let cs = self.rng.gen_f64() < self.c.coursera_share; + self.build_in.push(self.now); + let cs = self.c.fan_out || self.rng.gen_f64() < self.c.coursera_share; self.coursera.push(cs); self.wip += 1; self.in_system += 1; @@ -203,9 +317,14 @@ impl Sim { fn launch(&mut self, id: usize) { self.in_system -= 1; + let w = self.now - self.started_at[id]; if self.now >= self.warm { - self.sojourn.push(self.now - self.started_at[id]); + self.sojourn.push(w); + } + if self.recent_w.len() == W_WINDOW { + self.recent_w.pop_front(); } + self.recent_w.push_back(w); } fn leave_wip(&mut self) { @@ -216,16 +335,55 @@ impl Sim { } } + /// Enter the build queue (first pass or rework); `j.step` is 1. + fn enter_build(&mut self, j: Job) { + self.build_in[j.id] = self.now; + let m = self.c.build_min; + self.enqueue(BUILD, j, m); + } + + /// The builder is free; the build itself may still run on (eq 1 elapsed beyond eq 2 work). + fn after_build(&mut self, j: Job) { + let extra = self.c.build_elapsed_min - self.c.build_min; + if extra > 0.0 { + self.at(extra, &Ev::Delay(Job { step: 7, ..j })); + } else { + self.finish_build(j); + } + } + + fn finish_build(&mut self, j: Job) { + if self.now >= self.warm { + self.v2.build_elapsed += self.now - self.build_in[j.id]; + self.v2.build_elapsed_n += 1; + } + let m = self.c.design_min; + self.enqueue(DESIGN, Job { step: 2, ..j }, m); + } + + /// The distinctness gate: rework a share alpha, else stage or launch directly. + fn after_verify(&mut self, j: Job) { + let a = self.c.rework_alpha; + if a > 0.0 && self.rng.gen_f64() < a { + self.enter_build(Job { step: 1, ..j }); + } else if self.coursera[j.id] { + let m = self.c.stage_min; + self.enqueue(CHROME, Job { step: 5, ..j }, m); + } else { + self.at(15.0, &Ev::Delay(Job { step: 6, ..j })); + } + } + fn course_next(&mut self, j: Job) { let c = self.c.clone(); let nj = |step| Job { step, ..j }; match j.step { - 0 => self.enqueue(BUILD, nj(1), c.build_min), - 1 => self.enqueue(DESIGN, nj(2), c.design_min), + 0 => self.enter_build(nj(1)), + 1 => self.after_build(j), + 7 => self.finish_build(j), 2 => self.enqueue(RENDER, nj(3), c.render_min), 3 => self.enqueue(VERIFY, nj(4), c.verify_min), - 4 if self.coursera[j.id] => self.enqueue(CHROME, nj(5), c.stage_min), - 4 => self.at(15.0, &Ev::Delay(nj(6))), + 4 => self.after_verify(j), 5 => { if !self.c.batch_in_wip { self.leave_wip(); @@ -253,17 +411,25 @@ impl Sim { } } + /// Shorts run steps 0-3 (render, wait, upload); LinkedIn feed renders run steps 4-5. fn short_next(&mut self, j: Job) { let nj = |step| Job { step, ..j }; + let tv = self.c.vertical_render_min; match j.step { - 0 => self.enqueue(RENDER, nj(1), 5.0), + 0 => self.enqueue(RENDER, nj(1), tv), 1 => self.at(20.0, &Ev::Delay(nj(2))), 2 => self.enqueue(CHROME, nj(3), 5.0), - _ => { + 3 => { if self.now >= self.warm { self.shorts_done += 1; } } + 4 => self.enqueue(RENDER, nj(5), tv), + _ => { + if self.now >= self.warm { + self.v2.feeds_done += 1; + } + } } } @@ -289,6 +455,14 @@ impl Sim { ); self.at(DAY / self.c.shorts_per_day, &Ev::ShortTick); } + Ev::FeedTick => { + self.short_next(Job { + short: true, + id: 0, + step: 4, + }); + self.at(DAY / self.c.linkedin_renders_per_day, &Ev::FeedTick); + } Ev::Done(s, j) => { self.st[s].free += 1; self.dispatch(s); @@ -306,6 +480,31 @@ impl Sim { } } } + + fn result(&self, end: f64) -> CourseFlowResult { + let span = end - self.warm; + let n = self.sojourn.len() as f64; + let mut util = [0.0; 6]; + for (u, st) in util.iter_mut().zip(&self.st) { + *u = st.busy_min / (span * st.size as f64); + } + let courses_per_day = n / span * DAY; + let r = if self.c.fan_out { FAN_OUT_R } else { 1.0 }; + let per_h = |sum: f64, k: u64| sum / (k.max(1) as f64) / 60.0; + CourseFlowResult { + courses_per_day, + shorts_per_day: self.shorts_done as f64 / span * DAY, + lead_days: self.sojourn.iter().sum::() / n.max(1.0) / DAY, + l: self.l_area / span, + util, + mean_batch_wait_h: per_h(self.bw_sum, self.bw_n), + stage_util: self.v2.stage_busy / (span * self.c.chrome_lanes.max(1) as f64), + launches_per_day: courses_per_day * r, + feed_renders_per_day: self.v2.feeds_done as f64 / span * DAY, + t_b_work_h: per_h(self.v2.build_work, self.v2.build_work_n), + t_b_elapsed_h: per_h(self.v2.build_elapsed, self.v2.build_elapsed_n), + } + } } /// Run one seed for `days`, discarding the first `warmup_d`. @@ -316,38 +515,12 @@ pub fn run_course_flow( days: f64, warmup_d: f64, ) -> CourseFlowResult { - let size = [1, c.builders, c.design_lanes, c.gpus, 4, c.chrome_lanes]; - let mut s = Sim { - c: c.clone(), - rng: SimRng::new(seed), - sched: EventScheduler::new(), - now: 0.0, - warm: warmup_d * DAY, - st: size - .iter() - .map(|&n| Station { - free: n, - size: n, - hi: VecDeque::new(), - lo: VecDeque::new(), - busy_min: 0.0, - }) - .collect(), - wip: 0, - pending: 0, - started_at: Vec::new(), - coursera: Vec::new(), - batch_wait: Vec::new(), - bw_sum: 0.0, - bw_n: 0, - sojourn: Vec::new(), - shorts_done: 0, - in_system: 0, - l_area: 0.0, - last_t: 0.0, - }; + let mut s = Sim::new(c, seed, warmup_d); s.at(0.0, &Ev::Start); s.at(0.0, &Ev::ShortTick); + if c.linkedin_renders_per_day > 0.0 { + s.at(0.0, &Ev::FeedTick); + } s.at(7.0 * 60.0, &Ev::Batch); let end = days * DAY; while let Some(ev) = s.sched.next() { @@ -365,23 +538,10 @@ pub fn run_course_flow( s.step(&e); } } - let span = end - s.warm; - let n = s.sojourn.len() as f64; - let mut util = [0.0; 6]; - for (u, st) in util.iter_mut().zip(&s.st) { - *u = st.busy_min / (span * st.size as f64); - } - CourseFlowResult { - courses_per_day: n / span * DAY, - shorts_per_day: s.shorts_done as f64 / span * DAY, - lead_days: s.sojourn.iter().sum::() / n.max(1.0) / DAY, - l: s.l_area / span, - util, - mean_batch_wait_h: s.bw_sum / (s.bw_n.max(1) as f64) / 60.0, - } + s.result(end) } -/// Named FLOW-006 factory cases. +/// Named FLOW-006 factory cases, plus "factory-v2" (Content Factory v2, flow model B5). #[must_use] pub fn course_preset(name: &str) -> Option { let mut c = CourseFlowConfig { @@ -400,6 +560,12 @@ pub fn course_preset(name: &str) -> Option { outline_min: 90.0, verify_min: 15.0, batch_in_wip: false, + fan_out: false, + rework_alpha: 0.0, + trunk_first_w_days: None, + linkedin_renders_per_day: 0.0, + vertical_render_min: 5.0, + build_elapsed_min: 0.0, }; match name { "cal-today" => { @@ -411,6 +577,12 @@ pub fn course_preset(name: &str) -> Option { "nine-builders" => c.builders = 9, "coursera-24h" => c.stage_min = 1440.0, "render-300" => c.render_min = 300.0, + "factory-v2" => { + c.fan_out = true; + c.batch_in_wip = true; + c.trunk_first_w_days = Some(4.8); + c.linkedin_renders_per_day = 4.0; + } _ => return None, } Some(c) @@ -428,20 +600,28 @@ pub fn course_cases() -> [&'static str; 5] { ] } +fn add_scaled(a: &mut CourseFlowResult, r: &CourseFlowResult, w: f64) { + a.courses_per_day += r.courses_per_day * w; + a.shorts_per_day += r.shorts_per_day * w; + a.lead_days += r.lead_days * w; + a.l += r.l * w; + a.mean_batch_wait_h += r.mean_batch_wait_h * w; + a.stage_util += r.stage_util * w; + a.launches_per_day += r.launches_per_day * w; + a.feed_renders_per_day += r.feed_renders_per_day * w; + a.t_b_work_h += r.t_b_work_h * w; + a.t_b_elapsed_h += r.t_b_elapsed_h * w; + for i in 0..6 { + a.util[i] += r.util[i] * w; + } +} + /// The reference run: seeds 1..=5, 300 days, 20 days warm-up, averaged. #[must_use] pub fn run_course_reference(c: &CourseFlowConfig) -> CourseFlowResult { let mut a = CourseFlowResult::default(); for seed in 1..=5 { - let r = run_course_flow(c, seed, 300.0, 20.0); - a.courses_per_day += r.courses_per_day / 5.0; - a.shorts_per_day += r.shorts_per_day / 5.0; - a.lead_days += r.lead_days / 5.0; - a.l += r.l / 5.0; - a.mean_batch_wait_h += r.mean_batch_wait_h / 5.0; - for i in 0..6 { - a.util[i] += r.util[i] / 5.0; - } + add_scaled(&mut a, &run_course_flow(c, seed, 300.0, 20.0), 0.2); } a } @@ -454,26 +634,48 @@ pub fn apply_course_inputs(c: &mut CourseFlowConfig, v: &serde_json::Value) -> R let obj = v.as_object().ok_or("inputs must be a JSON object")?; for (k, x) in obj { let f = x.as_f64().ok_or_else(|| format!("{k}: not a number"))?; - match k.as_str() { - "builders" => c.builders = f as usize, - "build_min" => c.build_min = f, - "design_lanes" => c.design_lanes = f as usize, - "design_min" => c.design_min = f, - "gpus" => c.gpus = f as usize, - "render_min" => c.render_min = f, - "chrome_lanes" => c.chrome_lanes = f as usize, - "stage_min" => c.stage_min = f, - "coursera_share" => c.coursera_share = f, - "wip" => c.wip = f as usize, - "starts_per_day" => c.starts_per_day = f, - "shorts_per_day" => c.shorts_per_day = f, - "batch_in_wip" => c.batch_in_wip = f != 0.0, - _ => return Err(format!("unknown input: {k}")), + if !apply_v2_input(c, k, f) { + apply_base_input(c, k, f)?; } } Ok(()) } +fn apply_base_input(c: &mut CourseFlowConfig, k: &str, f: f64) -> Result<(), String> { + match k { + "builders" => c.builders = f as usize, + "build_min" => c.build_min = f, + "design_lanes" => c.design_lanes = f as usize, + "design_min" => c.design_min = f, + "gpus" => c.gpus = f as usize, + "render_min" => c.render_min = f, + "chrome_lanes" => c.chrome_lanes = f as usize, + "stage_min" => c.stage_min = f, + "coursera_share" => c.coursera_share = f, + "wip" => c.wip = f as usize, + "starts_per_day" => c.starts_per_day = f, + "shorts_per_day" => c.shorts_per_day = f, + "batch_in_wip" => c.batch_in_wip = f != 0.0, + _ => return Err(format!("unknown input: {k}")), + } + Ok(()) +} + +/// The Content Factory v2 inputs; returns false when `k` is not one of them. +fn apply_v2_input(c: &mut CourseFlowConfig, k: &str, f: f64) -> bool { + match k { + "fan_out" => c.fan_out = f != 0.0, + "rework_alpha" => c.rework_alpha = f, + "trunk_first_w_days" => c.trunk_first_w_days = (f > 0.0).then_some(f), + "linkedin_renders_per_day" => c.linkedin_renders_per_day = f, + "vertical_render_min" => c.vertical_render_min = f, + "t_b_work_h" => c.build_min = f * 60.0, + "t_b_elapsed_h" => c.build_elapsed_min = f * 60.0, + _ => return false, + } + true +} + /// The `sim` block fields this scenario owns, in the shape the cops publish. #[must_use] pub fn course_sim_block( @@ -491,10 +693,19 @@ pub fn course_sim_block( "chrome_lanes": c.chrome_lanes, "stage_min": c.stage_min, "coursera_share": c.coursera_share, "wip": c.wip, "starts_per_day": c.starts_per_day, "shorts_per_day": c.shorts_per_day, "batch_in_wip": c.batch_in_wip, + "fan_out": c.fan_out, "rework_alpha": c.rework_alpha, + "trunk_first_w_days": c.trunk_first_w_days.unwrap_or(0.0), + "linkedin_renders_per_day": c.linkedin_renders_per_day, + "vertical_render_min": c.vertical_render_min, + "t_b_work_h": c.build_min / 60.0, + "t_b_elapsed_h": c.build_elapsed_min.max(c.build_min) / 60.0, }, "predicted": { "courses_day": r.courses_per_day, "shorts_day": r.shorts_per_day, "lead_days": r.lead_days, "L": r.l, "mean_batch_wait_h": r.mean_batch_wait_h, "busiest_util_pct": util * 100.0, + "stage_util_pct": r.stage_util * 100.0, "launches_day": r.launches_per_day, + "feed_renders_day": r.feed_renders_per_day, + "t_b_work_h": r.t_b_work_h, "t_b_elapsed_h": r.t_b_elapsed_h, }, "binding_station": station, }) @@ -629,4 +840,238 @@ mod tests { fn unknown_case_is_none() { assert!(course_preset("nope").is_none()); } + + // ---- B5: Content Factory v2 (flow model eqs 2, 3, 5, 6, 14) ---- + + fn v2() -> CourseFlowConfig { + course_preset("factory-v2").expect("factory-v2 case") + } + + /// Flow model eq 6 law: lanes busy = B * t_sc / (1440 * lanes) = 5 * 360 / (1440 * 2) = 62.5%. + const STAGE_LAW: f64 = 5.0 * 360.0 / (DAY * 2.0); + + /// Flow model eq 2 law at N_b = 11, t_b = 42 h: 5 * 2520 / (1440 * 11) = 79.5%. + const BUILD_LAW: f64 = 5.0 * 2520.0 / (DAY * 11.0); + + /// Eq 6: with every master fanned out (no split) the stage matches the law within 2 points. + #[test] + fn coursera_stage_matches_the_law_within_two_points() { + let r = run_course_reference(&v2()); + assert!( + (r.stage_util - STAGE_LAW).abs() <= 0.02, + "stage sim {:.3} vs law {STAGE_LAW:.3}", + r.stage_util + ); + } + + /// The legacy split reproduces the recorded gap: share 0.4 of the law, not the law. + #[test] + fn the_legacy_split_is_the_recorded_gap() { + let r = got("design"); + assert!( + (r.stage_util - 0.4 * STAGE_LAW).abs() <= 0.02, + "{:.3}", + r.stage_util + ); + } + + /// Eq 3: launches/day = B * R with R = 3; the legacy split launches each course once. + #[test] + fn fan_out_launches_three_renditions_per_master() { + let r = run_course_reference(&v2()); + assert!((r.launches_per_day - 3.0 * r.courses_per_day).abs() < 1e-9); + assert!( + (r.courses_per_day - 4.92).abs() <= 4.92 * 0.05, + "{:.3}", + r.courses_per_day + ); + let legacy = got("design"); + assert!((legacy.launches_per_day - legacy.courses_per_day).abs() < 1e-9); + } + + /// Eq 14: at alpha = 0.16 the build busy fraction is 79.5% / (1 - alpha) <= 95%, within 2 points. + #[test] + fn rework_at_alpha_016_keeps_build_at_or_under_95_percent() { + let mut c = v2(); + c.rework_alpha = 0.16; + let r = run_course_reference(&c); + let law = BUILD_LAW / (1.0 - 0.16); + assert!(r.util[BUILD] <= 0.95, "build busy {:.3}", r.util[BUILD]); + assert!( + (r.util[BUILD] - law).abs() <= 0.02, + "build sim {:.3} vs law {law:.3}", + r.util[BUILD] + ); + } + + /// Eq 14 falsifier: alpha = 0 leaves the build at the eq 2 law, so the rework loop is what moved it. + #[test] + fn without_rework_build_sits_at_the_eq2_law() { + let r = run_course_reference(&v2()); + assert!( + (r.util[BUILD] - BUILD_LAW).abs() <= 0.02, + "{:.3}", + r.util[BUILD] + ); + } + + /// Eq 5: GPU busy = (B * t_g + (B * k + 4) * t_v) / 1440 with k = 1, t_v = 5 min. + #[test] + fn gpu_carries_trunk_shorts_and_four_linkedin_renders() { + let r = run_course_reference(&v2()); + let law = (5.0 * 100.0 + (5.0 + 4.0) * 5.0) / DAY; + assert!( + (r.util[RENDER] - law).abs() <= 0.02, + "gpu sim {:.3} vs law {law:.3}", + r.util[RENDER] + ); + assert!((r.feed_renders_per_day - 4.0).abs() <= 0.1); + assert!((r.shorts_per_day - 5.0).abs() <= 0.1); + } + + /// Which job a free server at `s` takes when a Short and a course both wait, given the + /// recent course W in days. True means the course. + fn picks_course(s: usize, rule: Option, w_days: f64) -> bool { + let mut c = v2(); + c.trunk_first_w_days = rule; + let mut sim = Sim::new(&c, 1, 0.0); + sim.recent_w.push_back(w_days * DAY); + let short = Job { + short: true, + id: 0, + step: 0, + }; + let course = Job { + short: false, + id: 1, + step: 3, + }; + sim.st[s].hi.push_back((short, 5.0)); + sim.st[s].lo.push_back((course, 100.0)); + !sim.next_job(s).expect("a waiting job").0.short + } + + /// GPU priority: the trunk goes first while W > 4.8 d, else Shorts; other stations and the + /// rule switched off keep Shorts first. + #[test] + fn gpu_serves_trunk_first_only_while_w_is_over_the_threshold() { + assert!( + picks_course(RENDER, Some(4.8), 5.0), + "W 5.0 > 4.8: trunk first" + ); + assert!(!picks_course(RENDER, Some(4.8), 4.5), "W 4.5: Shorts first"); + assert!(!picks_course(RENDER, None, 9.0), "rule off: Shorts first"); + assert!( + !picks_course(CHROME, Some(4.8), 9.0), + "stage lanes: Shorts first" + ); + } + + /// GPU priority reorders the queue and cannot add GPU minutes: Shorts are never dropped, so + /// on an overloaded GPU the courses/day is the same either way, and Shorts carry the wait. + #[test] + fn gpu_priority_conserves_gpu_minutes_under_overload() { + let mut on = v2(); + on.render_min = 300.0; + let mut off = on.clone(); + off.trunk_first_w_days = None; + let a = run_course_reference(&on); + let b = run_course_reference(&off); + assert!(a.util[RENDER] > 0.99 && b.util[RENDER] > 0.99); + assert!( + (a.courses_per_day - b.courses_per_day).abs() <= b.courses_per_day * 0.01, + "trunk-first {:.3} vs shorts-first {:.3}", + a.courses_per_day, + b.courses_per_day + ); + assert!( + a.shorts_per_day < b.shorts_per_day, + "{:.3}", + a.shorts_per_day + ); + } + + /// Eq 2 vs eq 1: the builder holds t_b_work; lead time carries t_b_elapsed. Longer elapsed + /// time must move lead time and leave build busy at the sizing law. + #[test] + fn build_work_sizes_the_station_and_elapsed_drives_lead_time() { + let base = run_course_reference(&v2()); + let mut c = v2(); + c.build_elapsed_min = 60.0 * 60.0; + let r = run_course_reference(&c); + assert!( + (r.t_b_work_h - 42.0).abs() <= 42.0 * 0.03, + "{:.2}", + r.t_b_work_h + ); + assert!( + (r.t_b_elapsed_h - 60.0).abs() <= 60.0 * 0.03, + "{:.2}", + r.t_b_elapsed_h + ); + assert!( + (r.util[BUILD] - BUILD_LAW).abs() <= 0.02, + "{:.3}", + r.util[BUILD] + ); + let dl = r.lead_days - base.lead_days; + assert!((dl - 0.75).abs() <= 0.15, "lead moved {dl:.3} d"); + assert!(base.t_b_elapsed_h >= base.t_b_work_h); + } + + /// The new inputs reach the model through the ledger and come back out in the sim block. + #[test] + fn v2_inputs_round_trip_through_the_ledger_and_sim_block() { + let mut c = course_preset("design").expect("case"); + let v = serde_json::json!({ + "fan_out": 1, "rework_alpha": 0.16, "trunk_first_w_days": 4.8, + "linkedin_renders_per_day": 4, "vertical_render_min": 6, + "t_b_work_h": 40, "t_b_elapsed_h": 50, + }); + apply_course_inputs(&mut c, &v).expect("inputs"); + assert!(c.fan_out); + assert!((c.rework_alpha - 0.16).abs() < 1e-12); + assert_eq!(c.trunk_first_w_days, Some(4.8)); + assert!((c.build_min - 2400.0).abs() < 1e-9); + assert!((c.build_elapsed_min - 3000.0).abs() < 1e-9); + let r = run_course_flow(&c, 1, 30.0, 5.0); + let b = course_sim_block("design", &c, &r); + for k in [ + "fan_out", + "rework_alpha", + "trunk_first_w_days", + "linkedin_renders_per_day", + "vertical_render_min", + "t_b_work_h", + "t_b_elapsed_h", + ] { + assert!(b["inputs"].get(k).is_some(), "inputs.{k}"); + } + for k in [ + "stage_util_pct", + "launches_day", + "t_b_work_h", + "t_b_elapsed_h", + ] { + assert!(b["predicted"].get(k).is_some(), "predicted.{k}"); + } + let mut off = c.clone(); + apply_course_inputs(&mut off, &serde_json::json!({"trunk_first_w_days": 0})).expect("off"); + assert_eq!(off.trunk_first_w_days, None); + } + + /// The v2 preset keeps the daily 07:00 batch and counts it inside WIP (flow model eq 1). + #[test] + fn v2_keeps_the_daily_batch_inside_wip() { + let c = v2(); + assert!(c.batch_in_wip && c.fan_out); + let r = run_course_reference(&c); + assert!(r.mean_batch_wait_h > 0.5 && r.mean_batch_wait_h < 24.0); + let lam_w = r.courses_per_day * r.lead_days; + assert!( + (lam_w - r.l).abs() <= r.l * 0.005, + "{lam_w:.3} vs {:.3}", + r.l + ); + } }