Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions contracts/apr-cli-operations-v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,11 @@ falsification_tests:
prediction: The timestamp in a cbtop report matches the UTC date at the moment it was generated, on both headless paths
test: commands::cbtop::tests::test_chrono_timestamp_is_current_utc_date
if_fails: Persisted CI provenance is stamped with a stale hardcoded date
- id: FALSIFY-OPS-013
rule: apr run prints every output mode and derives tok/s as documented
prediction: print_run_output succeeds for stream, json, benchmark and text; effective_tok_per_sec prefers the reported rate, else tokens/duration, else 0.0
test: commands::run::tests::print_run_output_serves_every_mode_and_derives_the_rate
if_fails: apr run output errors on a finished run or reports a wrong rate
kani_harnesses:
- id: KANI-OPS-001
obligation: ReadOnly commands have no side effects
Expand Down
28 changes: 28 additions & 0 deletions crates/apr-cli/src/commands/run_tests_accel_reconcile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -423,3 +423,31 @@ mod pmat3826_fell_back_names_a_real_fallback {
);
}
}

/// FALSIFY-OPS-013: every output mode of `print_run_output` completes on a
/// finished run, and the rate it reports is derived exactly as documented.
#[test]
fn print_run_output_serves_every_mode_and_derives_the_rate() {
let mut r = gpu_result(Some(true));
// stream, json, benchmark (text and json), plain text (cached, then downloaded)
for (fmt, bench, stream) in [
("text", false, true),
("json", false, false),
("text", true, false),
("json", true, false),
("text", false, false),
] {
assert!(print_run_output(&r, "model.gguf", fmt, 8, bench, stream, false).is_ok());
}
r.cached = false;
assert!(print_run_output(&r, "model.gguf", "text", 8, false, false, true).is_ok());

// reported rate wins; otherwise tokens / duration; zero duration gives 0.0
r.tok_per_sec = Some(7.5);
assert!((effective_tok_per_sec(&r, 4) - 7.5).abs() < f64::EPSILON);
r.tok_per_sec = None;
r.duration_secs = 2.0;
assert!((effective_tok_per_sec(&r, 4) - 2.0).abs() < f64::EPSILON);
r.duration_secs = 0.0;
assert!(effective_tok_per_sec(&r, 4).abs() < f64::EPSILON);
}
1 change: 1 addition & 0 deletions docs/findings/aprender-w5404.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"session":"aprender-w5404","ref":"cov-run-output","note":"pmat ticket not minted (needs GH issue); tests for run print_run_output/effective_tok_per_sec, FALSIFY-OPS-013"}
Loading