From 1e2fd5303997f948b53bc90a5c9e617d34e0eda4 Mon Sep 17 00:00:00 2001 From: Jairo Gelpi Date: Thu, 17 Sep 2026 14:29:14 +0200 Subject: [PATCH] fix: require recomputed evidence before supporting numeric claims --- README.md | 19 ++- benchmarks/external/README.md | 28 +++- docs/CLAIM_VERIFICATION.md | 74 ++++++++++ docs/VALIDATION.md | 17 +++ .../0003-closed-numeric-claim-verification.md | 45 ++++++ examples/campaign_retention/question.yaml | 2 +- requirements/traceability.yaml | 2 +- scripts/evaluate_external.py | 44 ++++-- scripts/verify_bike_claims.py | 121 +++++++++++++++ .../application/assessment_runner.py | 77 +++++++++- src/answerable/cli.py | 1 + src/answerable/evidence/verdict.py | 9 +- src/answerable/reports/markdown.py | 6 +- tests/e2e/test_runner_edge_cases.py | 4 +- tests/test_external_evaluation.py | 12 +- tests/test_numeric_claims.py | 138 ++++++++++++++++++ tests/test_validity_adversarial.py | 13 +- tests/test_verdict_and_claims.py | 4 +- 18 files changed, 578 insertions(+), 38 deletions(-) create mode 100644 docs/CLAIM_VERIFICATION.md create mode 100644 docs/adr/0003-closed-numeric-claim-verification.md create mode 100644 scripts/verify_bike_claims.py create mode 100644 tests/test_numeric_claims.py diff --git a/README.md b/README.md index 44dd18a..0895fba 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Your code has tests. Your data has tests. **Your conclusions should too.** > [!IMPORTANT] > **Status: alpha.** Answerable checks explicit data and analysis conditions within its implemented scope. Its reports separate verified facts, declared assumptions and conditions not verifiable from the supplied table. > -> CLI and MCP have automated tests. The external-data evaluation uses two datasets and 12 project-authored questions. Improvement from giving an agent Answerable has **not yet been demonstrated with real models**. See [validation evidence](docs/VALIDATION.md) and [stable release criteria](docs/RELEASE_CRITERIA.md). +> CLI and MCP have automated tests. The external-data evaluation uses two datasets and 20 project-authored cases in a restricted mean-statement format. Improvement from giving an agent Answerable has **not yet been demonstrated with real models**. See [validation evidence](docs/VALIDATION.md) and [stable release criteria](docs/RELEASE_CRITERIA.md). ![Answerable terminal demo](docs/demo.svg) @@ -74,7 +74,7 @@ reported as `ANSWERABLE_WITH_ASSUMPTIONS`, with facts and unverified conditions kept separate. See [ADR 0002](docs/adr/0002-explicit-identification-and-claim-blockers.md). The [external-data suite](benchmarks/external/README.md) records false blocks as -well as unsafe allowed claims. Its two datasets are external; its 12 questions +well as unsafe allowed claims. Its two datasets are external; its 20 restricted-format cases and labels are project-authored. This is a small regression evaluation, not proof of general validity. A paired agent/tool experiment is executable but **has not been run with real models in this revision**. @@ -84,6 +84,21 @@ fixtures now explicitly declare identifying assumptions and produce conditional baseline verdicts; they are not identical inputs to that historical comparison. A 100% score on project-authored rules/cases is not general superiority over LLMs. +## Claim verification boundary + +A Bike Sharing adversarial test exposed a defect: arbitrary false numbers could +appear in `allowed_claims`. The current source fix supports only **exact, +recomputed descriptive mean statements**. Free prose, even when true, is +unverified; declaring a causal assumption does not verify a causal estimate. +`ANSWERABLE` assesses the question's implemented prerequisites, not every +submitted sentence. Always inspect the claim lists and findings. + +See [the supported format and Windows retest](docs/CLAIM_VERIFICATION.md). +The extended external evaluation includes truthful free prose and reports its +false blocks: **4 of 8 valid claims blocked**, with **0 of 16 invalid claims +allowed** in the regression. This limitation prevents a claim of general +conclusion validation. Historical benchmark reports are not rewritten. + ## Quickstart ### 1. Install the published alpha package diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md index dbfa21f..4a93d3f 100644 --- a/benchmarks/external/README.md +++ b/benchmarks/external/README.md @@ -27,19 +27,35 @@ are encoded as UTC midnight for the runner; these are date labels, not claimed measurement times. Group 0 is January–June, group 1 July–December. Scope is either the complete series or its first year. No population generalization is requested. -Each of the four dataset/scope combinations has three questions: +Each of the four dataset/scope combinations has five restricted-format cases: 1. **Valid:** the two observed sample means, computed independently with - `statistics.mean`, with explicit population and units. + `statistics.mean`, in the documented canonical mean format; columns and scope are explicit. 2. **Invalid causal claim:** attributing that difference to assignment to the second half of the year, with no supplied identification evidence/assumptions. 3. **Invalid grain:** the descriptive candidate after intentionally duplicating one dated entity under a one-row-per-date contract. This is an explicit perturbation, not a claim that the original source has duplicate records. +4. **False number:** a group mean is replaced with 99999. +5. **Swapped means:** the correct values are assigned to the opposite groups. + +An optional sixth case keeps the original truthful free prose. It intentionally +measures valid claims that the closed verifier cannot support: + +```bash +python scripts/evaluate_external.py --output runs/external-free-text --include-free-text +``` + +This broader evaluation returns nonzero because four valid free-text claims are +blocked (4/8 valid; 0/16 unsafe allows). Do not ignore that failure in publication +claims. The default 20-case CI gate has zero false blocks in its narrower +supported grammar. Thresholds remain unchanged. Neither suite proves arbitrary +natural-language verification. Historical v1 results used different claims. + Labels, data hashes and question hashes are written before calling the runner. The labels/questions are project-authored, not externally peer-reviewed. Cases -share source data and are correlated. Do not treat 12 cases or repeated agent +share source data and are correlated. Do not treat 20 restricted-format cases or repeated agent calls as independent evidence of general reliability. ## Run the engine evaluation @@ -88,7 +104,7 @@ adapter must disable ambient tools and memory; the harness cannot enforce those properties inside an arbitrary supplied command. Record the exact provider model, not a moving alias, and verify that the adapter actually uses it. -The default is 12 cases × 2 repetitions × 2 conditions = 48 decisions. Both arms +The default is 20 restricted-format cases × 2 repetitions × 2 conditions = 80 decisions. Both arms get the same aggregate evidence and claim. Neither sees labels or source paths. Cases are deterministically shuffled and condition order counterbalanced. Every model invocation starts in a fresh temporary directory without prior conversation. @@ -113,7 +129,7 @@ The recorded preflight currently has **zero real model decisions**. ## Planned independent acceptance evaluation **Status: protocol only; not collected or executed.** The current scripts -materialize the 12 smoke cases above. They do not yet ingest a general independent +materialize the 20 restricted-format cases above. They do not yet ingest a general independent question bank, adjudicate labels, calculate dataset-cluster intervals, or fetch provider billing. Implement and validate those capabilities before using them for [stable-release acceptance](../../docs/RELEASE_CRITERIA.md). @@ -161,6 +177,6 @@ a false-block increase of no more than two percentage points and a complete run. A small/inconclusive interval or a baseline with zero unsafe allows does not establish benefit; retain the result and extend the evaluation prospectively. -The present 48-decision smoke plan does not satisfy this independent protocol. +The present 80-decision smoke plan does not satisfy this independent protocol. Passing test-double adapters verifies plumbing only. A native MCP host evaluation would be a separately specified experiment; do not conflate it with mediated calls. diff --git a/docs/CLAIM_VERIFICATION.md b/docs/CLAIM_VERIFICATION.md new file mode 100644 index 0000000..ce44a18 --- /dev/null +++ b/docs/CLAIM_VERIFICATION.md @@ -0,0 +1,74 @@ +# Numerical claim verification + +The runner recomputes means from the supplied CSV/Parquet file. A submitted +claim is supported only if its class is `descriptive`, its complete text equals +the canonical statement, and applicable blockers/lint checks permit it. There +is no caller-controlled `verified` flag in the question JSON. Extra prose is +not validated merely because part of a statement contains correct numbers. + +Example for UCI Bike Sharing `days-ready.csv`: + +```text +Observed means of "cnt" by "workingday" among supplied records (six decimals): "0"=4330.168831 (n=231); "1"=4584.820000 (n=500). +``` + +Column names and group labels use JSON string quoting, groups are sorted by +their string values, and numbers have exactly six decimal places. Verification +compares the complete generated text: this admits the rounded six-decimal +result, not arbitrary tolerances or alternative rounding. `n` counts non-null +numeric outcomes. Missing/non-numeric/non-finite outcomes or null groups block +support, rather than silently changing the denominator. The population is the +supplied records, without causal interpretation, automatic date filtering or +extrapolation. Prepare the desired scope upstream. One source is accepted; +multiple sources require an explicit upstream join. + +In JSON results and warrant `data_quality_relevance`, `computed_mean_claim` +contains the arithmetic candidate and `claim_validation` records each submitted +claim's status. A computed candidate is not permission to publish: check the +allowed list and all blockers. `unverified_or_blocked` does not mean disproved. +Arbitrary prose (including correct prose) and causal estimates remain unverified. +The question verdict can be `ANSWERABLE` with no supported submitted claims. + +## Windows retest + +From the existing checkout, update the source and editable installation: + +```powershell +git pull --ff-only +.\.venv\Scripts\python.exe -m pip install -e ".[dev]" +.\.venv\Scripts\answerable.exe assess --data datos\bikes\days-ready.csv --question datos\bikes\false-number.json --output runs\bikes-false-number-fixed +``` + +Expected: the 99999/1 statement appears only under unsupported claims, never +supported. Then create a canonical true candidate without overwriting the old +questions: + +```powershell +@' +import json +from pathlib import Path +folder = Path("datos/bikes") +q = json.loads((folder / "descriptive.json").read_text(encoding="utf-8")) +q["question_id"] = "bikes_verified_means" +q["claims"] = [{"claim_class": "descriptive", "text": 'Observed means of "cnt" by "workingday" among supplied records (six decimals): "0"=4330.168831 (n=231); "1"=4584.820000 (n=500).'}] +(folder / "verified-means.json").write_text(json.dumps(q, indent=2), encoding="utf-8") +'@ | .\.venv\Scripts\python.exe - +.\.venv\Scripts\answerable.exe assess --data datos\bikes\days-ready.csv --question datos\bikes\verified-means.json --output runs\bikes-verified-means +``` + +To run all six numerical regression cases automatically against the original file: + +```powershell +.\.venv\Scripts\python.exe scripts\verify_bike_claims.py --data datos\bikes\day.csv --output runs\bikes-regression +``` + +Expected: `passed: 6`, with the true statement supported and all five altered +statements unsupported. The script independently computes reference means and +verifies each generated warrant. Dataset attribution: Hadi Fanaee-T (2013), +UCI Bike Sharing, DOI 10.24432/C5W894, CC BY 4.0. + +Expected: that exact statement is supported. Change a value, group, count or +append another assertion: it must no longer be supported. Rerun duplicate-data +and warrant-tampering tests as before. Old warrants remain cryptographically +verifiable; a valid hash never retroactively proves a claim's truth. Regenerate +assessments created before this correction. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 7d85706..fa83bf9 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -97,3 +97,20 @@ Independent questions and labels, dataset-level held-out evaluation, real-model paired results, and external pilots remain pending. Windows jobs are introduced in the release-readiness change; inspect that revision's CI before claiming a pass. [Release criteria](RELEASE_CRITERIA.md) define the gates and their evidence owners. + +## Numeric-claim correction (2026-09-17) + +The earlier 12-case snapshot did not test false numerical text. A real Bike +Sharing test found that such text could be admitted. It does not constitute +validation of numerical claims. Current source verifies the closed descriptive +format in [CLAIM_VERIFICATION.md](CLAIM_VERIFICATION.md), with adversarial tests +for invented values, swapped groups, altered counts, columns and extra prose. +The default external suite now has 20 cases. The optional free-text extension +has 24 and deliberately exposes four false blocks among eight valid statements. +Keep these different protocols separate when reporting historical scores. + +The current power calculation already uses sample variances for numerical +outcomes; it is not a binomial-only calculation. Its observed-effect normal +approximation is not prospective design power and does not account for temporal +dependence in the bike series. Neither it nor the new claim verifier establishes +causality or stable product readiness. diff --git a/docs/adr/0003-closed-numeric-claim-verification.md b/docs/adr/0003-closed-numeric-claim-verification.md new file mode 100644 index 0000000..087a8bb --- /dev/null +++ b/docs/adr/0003-closed-numeric-claim-verification.md @@ -0,0 +1,45 @@ +# ADR 0003 — Closed numerical claim verification + +Status: implemented; full CI pending. + +## Context + +INV-001/002 were violated when claim-class linting admitted invented numeric +prose. Declared identification assumptions could also admit a causal sentence +without an actual causal estimator. A provenance edge alone does not validate +a sentence's contents. + +## Decision + +The runner verifies a closed canonical statement of computed descriptive group +means. Class linting and prerequisite gates are necessary but not sufficient. +VerdictEngine defaults to no verified candidates; the runner supplies only its +own computed summary. This is a trusted internal Python boundary, not an input +JSON verification flag. Causal, predictive and other prose is not certified. + +Reject invalid outcomes/groups and multiple sources rather than endorsing an +aggregate with an implicit denominator change or ignored input. Include the +verifier revision in assessment identity. Preserve higher-priority integrity +and identification verdicts; the question verdict and candidate validity are +separate. Explain this explicitly in CLI, Markdown and warrant limitations. + +## Compatibility + +Question and public v1 schemas stay unchanged. Legacy text is still loadable +but no longer automatically supported. New observation keys are additive in an +existing open mapping. Six-decimal rounding is fixed, not caller configurable. +Existing warrants are unchanged and remain hash-verifiable; verification proves +integrity, not substantive correctness. Reassess old data to apply this fix. + +Existing tests that permitted uncomputed causal estimates now assert refusal; +mean examples use independent expected values. Frozen benchmarks remain intact. +The external regression adds numeric falsification and swapped groups. Optional +true free text is still labelled valid and counted as false blocks, not relabelled +as invalid to improve a score. No quality gate or coverage threshold is lowered. + +## Limits + +This conservative verifier cannot validate arbitrary language, perform adjusted +causal estimation or establish population representativeness. Narrow-format +regression success is not stable-product validation. The extended evaluation +must accompany any claim about general natural-language performance. diff --git a/examples/campaign_retention/question.yaml b/examples/campaign_retention/question.yaml index d077e3b..11f68c4 100644 --- a/examples/campaign_retention/question.yaml +++ b/examples/campaign_retention/question.yaml @@ -43,7 +43,7 @@ causal: sensitivity_checks: [] claims: - - text: "Observed 90-day retention was 12 points higher among exposed customers." + - text: 'Observed means of "retained_90d" by "campaign_exposed" among supplied records (six decimals): "false"=0.480000 (n=25); "true"=0.600000 (n=25).' claim_class: descriptive - text: "The campaign caused a 12 point increase in 90-day retention." claim_class: causal diff --git a/requirements/traceability.yaml b/requirements/traceability.yaml index 144afa3..e142115 100644 --- a/requirements/traceability.yaml +++ b/requirements/traceability.yaml @@ -458,7 +458,7 @@ requirements: status: verified INV-001: implementation: ["src/answerable/application/assessment_runner.py"] - tests: ["tests/test_validity_adversarial.py"] + tests: ["tests/test_validity_adversarial.py", "tests/test_numeric_claims.py"] status: implemented INV-002: implementation: ["src/answerable/evidence/verdict.py"] diff --git a/scripts/evaluate_external.py b/scripts/evaluate_external.py index f004346..f716f55 100644 --- a/scripts/evaluate_external.py +++ b/scripts/evaluate_external.py @@ -24,7 +24,7 @@ def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def materialize(output: Path) -> list[dict[str, object]]: +def materialize(output: Path, *, include_free_text: bool = False) -> list[dict[str, object]]: """Transform external dates without inventing observations; preserve every value.""" manifest = json.loads((SUITE / "manifest.json").read_text()) cases = [] @@ -59,7 +59,16 @@ def materialize(output: Path) -> list[dict[str, object]]: ) a = mean(r["value"] for r in selected if r["group"] == 0) b = mean(r["value"] for r in selected if r["group"] == 1) - for variant in ("observed", "causal", "duplicate"): + for variant in ( + "observed", + "causal", + "duplicate", + "false_number", + "swapped_groups", + "unverified_text", + ): + if variant == "unverified_text" and not include_free_text: + continue case_id = f"{source['id']}-{scope}-{variant}" folder = output / "cases" / case_id folder.mkdir(parents=True, exist_ok=True) @@ -68,10 +77,23 @@ def materialize(output: Path) -> list[dict[str, object]]: writer = csv.DictWriter(stream, fieldnames=["entity", "date", "group", "value"]) writer.writeheader() writer.writerows(data) + n0 = sum(r["group"] == 0 for r in selected) + n1 = sum(r["group"] == 1 for r in selected) claim = ( - f"Among these recorded {source['unit']}s, the observed mean was {a:.6g} " - f"in January-June and {b:.6g} in July-December ({source['units']})." + 'Observed means of "value" by "group" among supplied records ' + f'(six decimals): "0"={a:.6f} (n={n0}); "1"={b:.6f} (n={n1}).' ) + if variant == "false_number": + claim = claim.replace(f'"0"={a:.6f}', '"0"=99999.000000') + if variant == "swapped_groups": + claim = claim.replace(f'"0"={a:.6f}', f'"0"={b:.6f}').replace( + f'"1"={b:.6f}', f'"1"={a:.6f}' + ) + if variant == "unverified_text": + claim = ( + f"Among these recorded {source['unit']}s, the observed mean was {a:.6g} " + f"in January-June and {b:.6g} in July-December ({source['units']})." + ) if variant == "causal": claim = ( "Assignment to July-December caused the observed difference in outcomes." @@ -119,7 +141,7 @@ def materialize(output: Path) -> list[dict[str, object]]: "case_id": case_id, "source": source["id"], "variant": variant, - "expected_allow": variant == "observed", + "expected_allow": variant in {"observed", "unverified_text"}, "claim": claim, "data_sha256": digest(folder / "data.csv"), "question_sha256": digest(folder / "question.json"), @@ -153,9 +175,9 @@ def engine_fingerprint() -> str: return hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() -def evaluate(output: Path) -> dict[str, object]: +def evaluate(output: Path, *, include_free_text: bool = False) -> dict[str, object]: output.mkdir(parents=True, exist_ok=True) - cases = materialize(output) + cases = materialize(output, include_free_text=include_free_text) # Freeze labels and hashes before invoking the engine; the oracle is not a verdict lookup. (output / "cases.json").write_text(json.dumps(cases, indent=2) + "\n") records = [] @@ -183,7 +205,7 @@ def evaluate(output: Path) -> dict[str, object]: except Exception as error: records.append({**case, "allowed": None, "error": f"{type(error).__name__}: {error}"}) report = { - "protocol": "external-smoke-v1", + "protocol": "external-claims-v2", "engine_sha256": engine_fingerprint(), "evaluator_sha256": digest(Path(__file__)), "dependencies": {name: version(name) for name in ("duckdb", "pyyaml", "sqlglot")}, @@ -192,7 +214,8 @@ def evaluate(output: Path) -> dict[str, object]: "metrics": metrics(records), "limitations": [ "Two external datasets; questions and labels authored within this project.", - "Duplicate cases are explicit perturbations of external data.", + "Duplicate, false-number and swapped-group cases are explicit perturbations.", + "Optional valid free-text claims measure the closed verifier false-block limitation.", "No population/causal generalization or independent expert validation.", "Cases within a source are correlated; no independence-based significance test.", ], @@ -205,8 +228,9 @@ def evaluate(output: Path) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--include-free-text", action="store_true") args = parser.parse_args() - report = evaluate(args.output) + report = evaluate(args.output, include_free_text=args.include_free_text) print(json.dumps(report["metrics"], indent=2)) scores = report["metrics"] return int(bool(scores["false_blocks"] or scores["unsafe_allows"] or scores["errors"])) diff --git a/scripts/verify_bike_claims.py b/scripts/verify_bike_claims.py new file mode 100644 index 0000000..3471cef --- /dev/null +++ b/scripts/verify_bike_claims.py @@ -0,0 +1,121 @@ +"""Reproduce the user-reported UCI Bike Sharing claim bug on a local day.csv.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from dataclasses import replace +from pathlib import Path +from statistics import mean + +from answerable.application.assessment_runner import AssessmentRunner +from answerable.application.models import ClaimCandidate +from answerable.application.spec_loader import load_spec +from answerable.evidence.claims import ClaimClass +from answerable.public import verify_warrant + + +def run(data: Path, output: Path) -> dict[str, object]: + output.mkdir(parents=True, exist_ok=True) + with data.open(encoding="utf-8", newline="") as stream: + rows = list(csv.DictReader(stream)) + assert len(rows) == 731, "Expected the original UCI day.csv (731 days)" + groups = {g: [int(r["cnt"]) for r in rows if r["workingday"] == g] for g in ("0", "1")} + assert [len(groups[g]) for g in ("0", "1")] == [231, 500] + assert mean(groups["1"]) == 4584.82 + for row in rows: + row["event_time"] = row["dteday"] + "T00:00:00+00:00" + source = output / "days-ready.csv" + with source.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + question = { + "question_id": "bikes_numeric_regression", + "raw_question": "Compare observed means in supplied days.", + "normalized_question": "Compare observed means in supplied days.", + "language": "en", + "analysis_type": "descriptive", + "unit_of_analysis": "day", + "population": {"description": "731 supplied days; no extrapolation."}, + "outcome": { + "metric_id": "rentals", + "definition": "Recorded rentals", + "value_type": "continuous", + }, + "time": { + "observation_start": "2011-01-01T00:00:00+00:00", + "observation_end": "2013-01-02T00:00:00+00:00", + }, + "data": { + "entity_column": "instant", + "event_time_column": "event_time", + "treatment_column": "workingday", + "outcome_column": "cnt", + "observation_window_days": 1, + "analysis_end": "2013-01-02T00:00:00+00:00", + }, + "causal": { + "treatment": "workingday", + "outcome": "cnt", + "population": "Recorded days", + "estimand": "Observed difference", + "strategy": "regression_adjustment", + }, + "claims": [], + } + spec_path = output / "question.json" + spec_path.write_text(json.dumps(question, indent=2), encoding="utf-8") + spec = load_spec(spec_path) + # Oracle uses Python statistics.mean, independently of DuckDB in the engine. + good = ( + 'Observed means of "cnt" by "workingday" among supplied records (six decimals): ' + f'"0"={mean(groups["0"]):.6f} (n=231); "1"={mean(groups["1"]):.6f} (n=500).' + ) + cases = { + "correct_means": (good, True), + "reported_false_prose": ( + "Among recorded days, mean rentals were 99999 on working days " + "and 1 on non-working days.", + False, + ), + "false_number": (good.replace("4584.820000", "99999.000000"), False), + "swapped_groups": ( + good.replace('"0"=', '"X"=').replace('"1"=', '"0"=').replace('"X"=', '"1"='), + False, + ), + "wrong_count": (good.replace("n=500", "n=499"), False), + "extra_causal_prose": (good + " Working days caused the increase.", False), + } + results = [] + for name, (text, expected) in cases.items(): + result = AssessmentRunner().run( + data_sources=(source,), + spec=replace(spec, claims=(ClaimCandidate(text, ClaimClass.DESCRIPTIVE),)), + output_directory=output / name, + ) + allowed = text in result.allowed_claims + assert allowed == expected, name + assert verify_warrant(result.warrant), name + results.append({"case": name, "expected_supported": expected, "supported": allowed}) + report = { + "source": "https://archive.ics.uci.edu/dataset/275/bike+sharing+dataset", + "source_sha256": hashlib.sha256(data.read_bytes()).hexdigest(), + "rows": len(rows), + "means": {g: mean(values) for g, values in groups.items()}, + "cases": results, + "passed": len(results), + "scope": "Numeric-claim regression, not full product or causal validation.", + } + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(run(args.data, args.output), indent=2)) diff --git a/src/answerable/application/assessment_runner.py b/src/answerable/application/assessment_runner.py index 5539fa4..ed3e6a5 100644 --- a/src/answerable/application/assessment_runner.py +++ b/src/answerable/application/assessment_runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path @@ -53,6 +54,7 @@ "timezone_ambiguity": "data_integrity", "prediction_leakage": "data_integrity", "duplicate_entities": "data_integrity", + "invalid_outcome": "data_integrity", "ambiguous_grain": "data_integrity", "insufficient_power": "power", "definition_change": "data_integrity", @@ -60,6 +62,7 @@ } _ALL_CLAIMS = frozenset( { + "invalid_outcome", "duplicate_entities", "prediction_leakage", "invalid_event_time", @@ -70,6 +73,12 @@ ) _DESIGN_IMPOSSIBLE = frozenset({"causal_identification_failure", "positivity_violation"}) _REPAIR = { + "invalid_outcome": ( + "Complete finite numeric outcomes and non-null group labels.", + "Silently excluding invalid records changes the population and denominator.", + True, + "correct the source values or document a new upstream analysis population", + ), "identification_evidence_missing": ( "Explicit design evidence or declared identifying assumptions.", "Empirical overlap alone does not establish causal identification.", @@ -178,6 +187,7 @@ _FINDING_CHECK = { + "invalid_outcome": "chk_observed_means", "duplicate_entities": "chk_grain_uniqueness", "ambiguous_grain": "chk_grain_uniqueness", "invalid_event_time": "chk_temporal_maturity", @@ -328,8 +338,8 @@ def run( spec: AssessmentSpec, output_directory: Path, ) -> AssessmentRun: - if not data_sources: - raise ValueError("at least one data source is required") + if len(data_sources) != 1: + raise ValueError("exactly one data source is supported per assessment") inspector = FileInspector() try: snapshots = tuple(inspector.inspect(path) for path in data_sources) @@ -342,6 +352,7 @@ def run( { "spec": to_dict(spec), "sources": [item.fingerprint for item in snapshots], + "claim_verifier": "observed-means-v1", } )[:16] ) @@ -375,12 +386,33 @@ def run( ) for candidate in spec.claims ) - verdict = VerdictEngine().decide(finding_inputs, claims=claims) + summary = _mean_claim(descriptive, spec.mapping) + verdict = VerdictEngine().decide( + finding_inputs, claims=claims, verified_descriptive_claims=(summary,) + ) repairs = _repair_plan(findings) status = _evidence_status(spec, rows, findings) graph = _build_graph(spec, snapshots, descriptive, findings, verdict, status) observations = { + "computed_mean_claim": summary, + "claim_validation": [ + { + "text": candidate.text, + "status": "supported" + if candidate.claim_class is ClaimClass.DESCRIPTIVE + and candidate.text in verdict.allowed_claims + else "unverified_or_blocked", + "reason": ( + "Matches recomputed means and passed applicable checks." + if candidate.claim_class is ClaimClass.DESCRIPTIVE + and candidate.text in verdict.allowed_claims + else "Not a verified descriptive mean statement, or blocked by findings. " + "Unsupported does not by itself mean false." + ), + } + for candidate in spec.claims + ], "identified": identified, "outcome_rates": [ {"group": group, "entities": count, "rate": rate} @@ -499,9 +531,12 @@ def _describe(path: Path, mapping: DataMapping) -> _Descriptive: outcome = _identifier(mapping.outcome_column) with duckdb.connect() as connection: rows = connection.execute( - f"SELECT cast({treatment} AS VARCHAR), count(*), avg(try_cast({outcome} AS DOUBLE)) " + f"SELECT cast({treatment} AS VARCHAR), count(try_cast({outcome} AS DOUBLE)), " + f"avg(try_cast({outcome} AS DOUBLE)) " f"FROM {_relation(path)} GROUP BY 1 ORDER BY 1" ).fetchall() + if any(row[2] is None or not math.isfinite(row[2]) for row in rows): + return _Descriptive((), None, None) by_group = tuple((str(row[0]), int(row[1]), float(row[2])) for row in rows) if len(by_group) != 2: return _Descriptive(by_group, None, None) @@ -509,6 +544,18 @@ def _describe(path: Path, mapping: DataMapping) -> _Descriptive: return _Descriptive(by_group, by_group[1][2] - baseline, baseline) +def _mean_claim(descriptive: _Descriptive, mapping: DataMapping) -> str: + """Closed, six-decimal grammar; never infer the meaning of arbitrary prose.""" + groups = "; ".join( + f"{json.dumps(group)}={rate:.6f} (n={count})" for group, count, rate in descriptive.by_group + ) + return ( + f"Observed means of {json.dumps(mapping.outcome_column)} by " + f"{json.dumps(mapping.treatment_column)} among supplied records " + f"(six decimals): {groups}." + ) + + def _overlap(rows: tuple[dict[str, object], ...], mapping: DataMapping) -> bool: """Conservative binary support check over every declared target stratum. @@ -536,6 +583,19 @@ def _run_checks( spec: AssessmentSpec, ) -> tuple[Finding, ...]: findings: list[Finding] = [] + if not rows or any( + not isinstance(row["outcome"], (int, float)) + or not math.isfinite(row["outcome"]) + or row["treatment"] is None + for row in rows + ): + findings.append( + Finding( + "invalid_outcome", + Severity.BLOCKER, + "Means require finite numeric outcomes, non-null groups, and non-empty data.", + ) + ) grain = GrainAnalyzer().infer(snapshot.profile) entities = {row["entity"] for row in rows} if len(entities) != len(rows): @@ -648,7 +708,7 @@ def _power_findings(rows: tuple[dict[str, object], ...]) -> tuple[Finding, ...]: by_treatment: dict[object, list[float]] = {} for row in rows: outcome = row.get("outcome") - if isinstance(outcome, (int, float)): + if isinstance(outcome, (int, float)) and math.isfinite(outcome): by_treatment.setdefault(row["treatment"], []).append(float(outcome)) groups = sorted(by_treatment.items(), key=lambda item: str(item[0])) if len(groups) != 2 or any(len(values) < 2 for _, values in groups): @@ -849,7 +909,12 @@ def _warrant_payload( ) ) ), - "limitations": [item.message for item in findings if item.severity is Severity.WARNING], + "limitations": [ + "The verdict assesses the question, not the truth of submitted prose. " + "Only exact recomputed descriptive mean statements can be supported; " + "free text and causal estimates are not verified.", + *[item.message for item in findings if item.severity is Severity.WARNING], + ], "data_quality_relevance": observations, "minimum_evidence_plan": { "candidates": [to_dict(item) for item in candidates], diff --git a/src/answerable/cli.py b/src/answerable/cli.py index be94e70..6e1d765 100644 --- a/src/answerable/cli.py +++ b/src/answerable/cli.py @@ -80,6 +80,7 @@ def _print_run(run: AssessmentRun) -> None: if run.blockers: for blocker in run.blockers: print(f" x {blocker.finding_id}: {blocker.message}") + print("Claim verification: only recomputed descriptive statements; free text is unverified.") print("Supported claims:") if run.allowed_claims: for claim in run.allowed_claims: diff --git a/src/answerable/evidence/verdict.py b/src/answerable/evidence/verdict.py index f998ac1..000ea0d 100644 --- a/src/answerable/evidence/verdict.py +++ b/src/answerable/evidence/verdict.py @@ -37,6 +37,7 @@ def decide( findings: tuple[FindingInput, ...], *, claims: tuple[tuple[str, ClaimContext], ...] = (), + verified_descriptive_claims: tuple[str, ...] = (), ) -> VerdictResult: blockers = tuple(item for item in findings if item.severity in {"blocker", "fatal"}) if any(item.repairability is None for item in blockers): @@ -46,9 +47,11 @@ def decide( forbidden: list[str] = [] linter = ClaimLinter() for claim, context in claims: - if linter.lint(claim, context) or any( - self.blocks_claim(item, context.claim_class) for item in blockers - ): + if ( + context.claim_class is not ClaimClass.DESCRIPTIVE + or claim not in verified_descriptive_claims + or linter.lint(claim, context) + ) or any(self.blocks_claim(item, context.claim_class) for item in blockers): forbidden.append(claim) else: allowed.append(claim) diff --git a/src/answerable/reports/markdown.py b/src/answerable/reports/markdown.py index bd194c0..3fca01d 100644 --- a/src/answerable/reports/markdown.py +++ b/src/answerable/reports/markdown.py @@ -32,9 +32,13 @@ def render_markdown(run: AssessmentRun) -> str: "", data["executive_explanation"], "", + "The verdict concerns question answerability. " + "Unsupported claims are unverified or blocked; " + "they are not necessarily false. Only exact recomputed descriptive means are supported.", + "", "## What you may claim", "", - _bullets(list(run.allowed_claims), "Nothing beyond the raw counts."), + _bullets(list(run.allowed_claims), "No submitted claim is verified and permitted."), "", "## What you may not claim", "", diff --git a/tests/e2e/test_runner_edge_cases.py b/tests/e2e/test_runner_edge_cases.py index b21f2d7..8418c6b 100644 --- a/tests/e2e/test_runner_edge_cases.py +++ b/tests/e2e/test_runner_edge_cases.py @@ -100,8 +100,8 @@ def test_supported_design_opens_the_causal_gate(self) -> None: ) self.assertEqual(run.verdict, Verdict.ANSWERABLE_WITH_ASSUMPTIONS) self.assertEqual(run.blockers, ()) - self.assertEqual(run.forbidden_claims, ()) - self.assertEqual(run.allowed_claims, ("Exposure caused higher 90-day retention.",)) + self.assertEqual(run.allowed_claims, ()) + self.assertEqual(run.forbidden_claims, ("Exposure caused higher 90-day retention.",)) markdown = (root / "out" / "warrant.md").read_text(encoding="utf-8") self.assertIn("Nothing blocks this question.", markdown) self.assertIn("Declared assumptions (not verified)", markdown) diff --git a/tests/test_external_evaluation.py b/tests/test_external_evaluation.py index 2cb6c5b..fefa4e5 100644 --- a/tests/test_external_evaluation.py +++ b/tests/test_external_evaluation.py @@ -16,9 +16,9 @@ def test_BENCH_external_data_keeps_valid_claims_and_rejects_invalid(tmp_path: Path) -> None: report = evaluate(tmp_path) assert report["metrics"] == { - "total": 12, + "total": 20, "valid_claims": 4, - "invalid_claims": 8, + "invalid_claims": 16, "false_blocks": 0, "unsafe_allows": 0, "false_block_rate": 0.0, @@ -106,3 +106,11 @@ def test_BENCH_paired_scores_include_regressions_and_exclude_incomplete_pairs() assert result["complete_pairs"] == 2 assert result["improved"] == result["regressed"] == 1 assert result["accuracy_delta"] == 0 + + +def test_BENCH_free_text_false_blocks_remain_visible(tmp_path: Path) -> None: + scores = evaluate(tmp_path, include_free_text=True)["metrics"] + assert scores["total"] == 24 + assert scores["valid_claims"] == 8 + assert scores["false_blocks"] == scores["release_false_blocks"] == 4 + assert scores["unsafe_allows"] == scores["errors"] == 0 diff --git a/tests/test_numeric_claims.py b/tests/test_numeric_claims.py new file mode 100644 index 0000000..dc65e9e --- /dev/null +++ b/tests/test_numeric_claims.py @@ -0,0 +1,138 @@ +"""INV-001/002: submitted prose is not evidence of its own truth.""" + +from dataclasses import replace +from pathlib import Path + +import pytest + +from answerable.application.assessment_runner import AssessmentRunner +from answerable.application.models import ClaimCandidate +from answerable.application.spec_loader import load_spec +from answerable.domain.models import AnalysisType, Verdict +from answerable.evidence.claims import ClaimClass +from tests.e2e.test_runner_edge_cases import _case + + +@pytest.mark.parametrize( + "text", + [ + "Among recorded days, mean rentals were 99999 on working days and 1 on non-working days.", + "Observed group means were 99999 and 1.", + "Exposure improved retention.", + ], +) +def test_INV_002_unverified_text_is_never_supported(tmp_path: Path, text: str) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + spec = load_spec(root / "question.json") + spec = replace( + spec, + contract=replace(spec.contract, analysis_type=AnalysisType.DESCRIPTIVE), + claims=(ClaimCandidate(text, ClaimClass.DESCRIPTIVE),), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert text not in run.allowed_claims + assert text in run.forbidden_claims + + +def test_INV_001_exact_means_preserve_valid_claims_and_reject_mutations(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + spec = load_spec(root / "question.json") + good = ( + 'Observed means of "retained_90d" by "campaign_exposed" among supplied records ' + '(six decimals): "false"=0.500000 (n=4); "true"=0.750000 (n=4).' + ) + bad = [ + good.replace("0.750000", "99999.000000"), + good.replace('"false"=0.500000', '"false"=0.750000').replace( + '"true"=0.750000', '"true"=0.500000' + ), + good.replace("(n=4)", "(n=5)"), + good + " This proves causation.", + good.replace("retained_90d", "revenue"), + good.replace("0.750000", "0.750001"), + ] + spec = replace( + spec, + contract=replace(spec.contract, analysis_type=AnalysisType.DESCRIPTIVE), + claims=tuple(ClaimCandidate(t, ClaimClass.DESCRIPTIVE) for t in [good, *bad]), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert run.allowed_claims == (good,) + assert run.forbidden_claims == tuple(bad) + assert run.warrant.data["allowed_claims"] == [good] + assert run.observations["computed_mean_claim"] == good + assert run.observations["claim_validation"][0]["status"] == "supported" + import json + + graph = json.loads((root / "out/evidence_graph.json").read_text()) + assert [n["payload"]["text"] for n in graph["nodes"] if n["type"] == "allowed_claim"] == [good] + + +@pytest.mark.parametrize("value", ["", "oops", "NaN", "inf", "-inf"]) +def test_INV_002_invalid_outcome_cannot_certify_a_mean(tmp_path: Path, value: str) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + source = root / "customers.csv" + lines = source.read_text().splitlines() + lines[1] = lines[1].rsplit(",", 1)[0] + "," + value + source.write_text("\n".join(lines) + "\n") + run = AssessmentRunner().run( + data_sources=(source,), + spec=load_spec(root / "question.json"), + output_directory=root / "out", + ) + assert not run.allowed_claims + assert run.verdict is Verdict.DATA_INTEGRITY_FAILURE + assert "invalid_outcome" in {b.finding_id for b in run.blockers} + + +def test_INV_001_additional_sources_are_not_silently_ignored(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + with pytest.raises(ValueError, match="exactly one data source"): + AssessmentRunner().run( + data_sources=(root / "customers.csv", root / "customers.csv"), + spec=load_spec(root / "question.json"), + output_directory=root / "out", + ) + + +def test_INV_001_rounding_and_integrity_are_independent(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + source = root / "customers.csv" + lines = source.read_text().splitlines() + lines[1:] = [line.rsplit(",", 1)[0] + ",1.2345674" for line in lines[1:]] + source.write_text("\n".join(lines) + "\n") + spec = load_spec(root / "question.json") + good = ( + 'Observed means of "retained_90d" by "campaign_exposed" among supplied records ' + '(six decimals): "false"=1.234567 (n=4); "true"=1.234567 (n=4).' + ) + spec = replace( + spec, + contract=replace(spec.contract, analysis_type=AnalysisType.DESCRIPTIVE), + claims=(ClaimCandidate(good, ClaimClass.DESCRIPTIVE),), + ) + run = AssessmentRunner().run(data_sources=(source,), spec=spec, output_directory=root / "out") + assert run.allowed_claims == (good,) + source.write_text("\n".join([*lines, lines[1]]) + "\n") + duplicate = good.replace('"true"=1.234567 (n=4)', '"true"=1.234567 (n=5)') + spec = replace(spec, claims=(ClaimCandidate(duplicate, ClaimClass.DESCRIPTIVE),)) + run = AssessmentRunner().run(data_sources=(source,), spec=spec, output_directory=root / "dup") + assert run.observations["computed_mean_claim"] == duplicate + assert run.verdict is Verdict.DATA_INTEGRITY_FAILURE + assert run.allowed_claims == () + + +def test_INV_002_null_group_blocks_even_matching_arithmetic(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + source = root / "customers.csv" + source.write_text(source.read_text().replace(",true,", ",,", 1)) + run = AssessmentRunner().run( + data_sources=(source,), + spec=load_spec(root / "question.json"), + output_directory=root / "out", + ) + assert run.verdict is Verdict.DATA_INTEGRITY_FAILURE diff --git a/tests/test_validity_adversarial.py b/tests/test_validity_adversarial.py index 0d1813d..7e03ad0 100644 --- a/tests/test_validity_adversarial.py +++ b/tests/test_validity_adversarial.py @@ -84,13 +84,20 @@ def test_INV_009_descriptive_question_does_not_require_causal_identification( spec = replace( spec, contract=replace(spec.contract, analysis_type=AnalysisType.DESCRIPTIVE), - claims=(ClaimCandidate("Observed group means differ.", ClaimClass.DESCRIPTIVE),), + claims=( + ClaimCandidate( + 'Observed means of "retained_90d" by "campaign_exposed" ' + "among supplied records (six decimals): " + '"false"=0.500000 (n=4); "true"=0.750000 (n=4).', + ClaimClass.DESCRIPTIVE, + ), + ), ) run = AssessmentRunner().run( data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" ) assert run.verdict is Verdict.ANSWERABLE - assert run.allowed_claims == ("Observed group means differ.",) + assert run.allowed_claims == (spec.claims[0].text,) def test_INV_001_findings_point_to_the_check_that_computed_them(tmp_path: Path) -> None: @@ -122,7 +129,7 @@ def test_INV_008_assumptions_never_become_facts_and_change_identity(tmp_path: Pa ) assert old.assessment_id != run.assessment_id assert run.verdict is Verdict.ANSWERABLE_WITH_ASSUMPTIONS - assert run.allowed_claims + assert not run.allowed_claims # Assumptions do not verify an uncomputed causal estimate. ledger = run.observations["evidence_status"] assert "conditional_exchangeability" in ledger["declared_assumptions"] assert ledger["unverifiable_conditions"] diff --git a/tests/test_verdict_and_claims.py b/tests/test_verdict_and_claims.py index 1918508..eb4650e 100644 --- a/tests/test_verdict_and_claims.py +++ b/tests/test_verdict_and_claims.py @@ -54,7 +54,9 @@ def test_phase_13_claim_linter_separates_allowed_and_forbidden(self) -> None: good = ClaimContext(ClaimClass.DESCRIPTIVE, "customers", "2026") bad = ClaimContext(ClaimClass.ASSOCIATION, None, None, causal_gate=False) result = VerdictEngine().decide( - (), claims=(("Revenue rose", good), ("Campaign caused revenue", bad)) + (), + claims=(("Revenue rose", good), ("Campaign caused revenue", bad)), + verified_descriptive_claims=("Revenue rose",), ) self.assertEqual(result.allowed_claims, ("Revenue rose",)) self.assertEqual(result.forbidden_claims, ("Campaign caused revenue",))