diff --git a/contracts/apr-cli-operations-v1.yaml b/contracts/apr-cli-operations-v1.yaml index 96ec9fb715..bc75c44a70 100644 --- a/contracts/apr-cli-operations-v1.yaml +++ b/contracts/apr-cli-operations-v1.yaml @@ -265,6 +265,11 @@ falsification_tests: prediction: The timestamp in a cbtop report matches the UTC date at the moment it was generated, on both headless paths test: commands::cbtop::tests::test_chrono_timestamp_is_current_utc_date if_fails: Persisted CI provenance is stamped with a stale hardcoded date +- id: FALSIFY-OPS-013 + rule: apr run prints every output mode and derives tok/s as documented + prediction: print_run_output succeeds for stream, json, benchmark and text; effective_tok_per_sec prefers the reported rate, else tokens/duration, else 0.0 + test: commands::run::tests::print_run_output_serves_every_mode_and_derives_the_rate + if_fails: apr run output errors on a finished run or reports a wrong rate kani_harnesses: - id: KANI-OPS-001 obligation: ReadOnly commands have no side effects diff --git a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs index 2ccb78c6d0..6b39720a66 100644 --- a/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs +++ b/crates/apr-cli/src/commands/run_tests_accel_reconcile.rs @@ -423,3 +423,31 @@ mod pmat3826_fell_back_names_a_real_fallback { ); } } + +/// FALSIFY-OPS-013: every output mode of `print_run_output` completes on a +/// finished run, and the rate it reports is derived exactly as documented. +#[test] +fn print_run_output_serves_every_mode_and_derives_the_rate() { + let mut r = gpu_result(Some(true)); + // stream, json, benchmark (text and json), plain text (cached, then downloaded) + for (fmt, bench, stream) in [ + ("text", false, true), + ("json", false, false), + ("text", true, false), + ("json", true, false), + ("text", false, false), + ] { + assert!(print_run_output(&r, "model.gguf", fmt, 8, bench, stream, false).is_ok()); + } + r.cached = false; + assert!(print_run_output(&r, "model.gguf", "text", 8, false, false, true).is_ok()); + + // reported rate wins; otherwise tokens / duration; zero duration gives 0.0 + r.tok_per_sec = Some(7.5); + assert!((effective_tok_per_sec(&r, 4) - 7.5).abs() < f64::EPSILON); + r.tok_per_sec = None; + r.duration_secs = 2.0; + assert!((effective_tok_per_sec(&r, 4) - 2.0).abs() < f64::EPSILON); + r.duration_secs = 0.0; + assert!(effective_tok_per_sec(&r, 4).abs() < f64::EPSILON); +} diff --git a/docs/findings/aprender-w5404.jsonl b/docs/findings/aprender-w5404.jsonl new file mode 100644 index 0000000000..915dd17612 --- /dev/null +++ b/docs/findings/aprender-w5404.jsonl @@ -0,0 +1 @@ +{"session":"aprender-w5404","ref":"cov-run-output","note":"pmat ticket not minted (needs GH issue); tests for run print_run_output/effective_tok_per_sec, FALSIFY-OPS-013"}