From 381ab99845055422093a57e23bb13f72aebc6b29 Mon Sep 17 00:00:00 2001 From: Jairo Gelpi Date: Tue, 15 Sep 2026 01:13:04 +0200 Subject: [PATCH] fix: separate identifying assumptions and enforce claim blockers --- README.md | 20 + benchmarks/external/README.md | 106 + benchmarks/external/manifest.json | 23 + .../external/results/2026-09-14/README.md | 50 + .../external/results/2026-09-14/after.json | 301 ++ .../2026-09-14/agent-preflight-manifest.json | 168 + .../results/2026-09-14/agent-preflight.json | 12 + .../external/results/2026-09-14/before.json | 301 ++ .../external/results/2026-09-14/cases.json | 158 + .../results/2026-09-14/verification.txt | 14 + benchmarks/external/sources/flights.csv | 145 + benchmarks/external/sources/temperatures.csv | 3651 +++++++++++++++++ ...licit-identification-and-claim-blockers.md | 71 + requirements/traceability.yaml | 20 + schemas/v1/causal-contract.schema.json | 87 +- scripts/compare_external_agents.py | 245 ++ scripts/evaluate_external.py | 216 + .../application/assessment_runner.py | 281 +- src/answerable/application/spec_loader.py | 1 + src/answerable/causal/contract.py | 10 + src/answerable/domain/serialization.py | 8 + src/answerable/evidence/claims.py | 2 + src/answerable/evidence/graph.py | 1 + src/answerable/evidence/verdict.py | 23 +- src/answerable/mutation_benchmark.py | 1 + src/answerable/reports/markdown.py | 19 +- tests/e2e/test_runner_edge_cases.py | 14 +- tests/test_external_evaluation.py | 108 + tests/test_mutation_benchmark.py | 4 +- tests/test_validity_adversarial.py | 217 + 30 files changed, 6222 insertions(+), 55 deletions(-) create mode 100644 benchmarks/external/README.md create mode 100644 benchmarks/external/manifest.json create mode 100644 benchmarks/external/results/2026-09-14/README.md create mode 100644 benchmarks/external/results/2026-09-14/after.json create mode 100644 benchmarks/external/results/2026-09-14/agent-preflight-manifest.json create mode 100644 benchmarks/external/results/2026-09-14/agent-preflight.json create mode 100644 benchmarks/external/results/2026-09-14/before.json create mode 100644 benchmarks/external/results/2026-09-14/cases.json create mode 100644 benchmarks/external/results/2026-09-14/verification.txt create mode 100644 benchmarks/external/sources/flights.csv create mode 100644 benchmarks/external/sources/temperatures.csv create mode 100644 docs/adr/0002-explicit-identification-and-claim-blockers.md create mode 100644 scripts/compare_external_agents.py create mode 100644 scripts/evaluate_external.py create mode 100644 tests/test_external_evaluation.py create mode 100644 tests/test_validity_adversarial.py diff --git a/README.md b/README.md index 5961a41..d8be523 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,26 @@ Verdict + blockers + allowed claims + forbidden claims Evidence Warrant ``` +## Validity review and external evaluation + +A targeted review corrected four issues: overlap no longer stands in for +exchangeability, all target strata must have empirical support, blockers now +apply to claim admissibility, and findings link to the checks that produced them. +Identifying assumptions must be explicitly declared; conditional support is +reported as `ANSWERABLE_WITH_ASSUMPTIONS`, with facts and unverified conditions +kept separate. See [ADR 0002](docs/adr/0002-explicit-identification-and-claim-blockers.md). + +The [external-data suite](benchmarks/external/README.md) records false blocks as +well as unsafe allowed claims. Its two datasets are external; its 12 questions +and labels are project-authored. This is a small regression evaluation, not proof +of general validity. A paired agent/tool experiment is executable but **has not +been run with real models in this revision**. + +The historical EMT-v2 scores below remain historical frozen results. The live +fixtures now explicitly declare identifying assumptions and produce conditional +baseline verdicts; they are not identical inputs to that historical comparison. +A 100% score on project-authored rules/cases is not general superiority over LLMs. + ## Quickstart ### 1. Install the production package diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md new file mode 100644 index 0000000..2996b89 --- /dev/null +++ b/benchmarks/external/README.md @@ -0,0 +1,106 @@ +# External-data validation and paired agent experiment + +This suite is deliberately separate from EMT. It tests claim admissibility, +including retention of valid descriptive claims, on data not generated by +Answerable. It is a **smoke evaluation, not independent scientific validation**. + +## Data and labels + +`manifest.json` records source URLs, retrieval date, SHA-256 and normalization. +Vendored CSV snapshots reproduce the run without a network request. + +- `flights`: 144 monthly observations from the seaborn example-data repository, + the international airline-passenger series also distributed as R AirPassengers. + Source: https://github.com/mwaskom/seaborn-data/blob/master/flights.csv +- `temperatures`: 3,650 daily minimum-temperature observations from the public + time-series dataset collection maintained by Jason Brownlee. + Source: https://github.com/jbrownlee/Datasets/blob/master/daily-min-temperatures.csv + +Both CSVs contain aggregate numerical measurements rather than personal records. +The source files are normalized to UTF-8/LF. Values are unchanged. Calendar dates +are encoded as UTC midnight for the runner; these are date labels, not claimed +measurement times. Group 0 is January–June, group 1 July–December. Scope is either +the complete series or its first year. No population generalization is requested. + +Each of the four dataset/scope combinations has three questions: + +1. **Valid:** the two observed sample means, computed independently with + `statistics.mean`, with explicit population and units. +2. **Invalid causal claim:** attributing that difference to assignment to the + second half of the year, with no supplied identification evidence/assumptions. +3. **Invalid grain:** the descriptive candidate after intentionally duplicating + one dated entity under a one-row-per-date contract. This is an explicit + perturbation, not a claim that the original source has duplicate records. + +Labels, data hashes and question hashes are written before calling the runner. +The labels/questions are project-authored, not externally peer-reviewed. Cases +share source data and are correlated. Do not treat 12 cases or repeated agent +calls as independent evidence of general reliability. + +## Run the engine evaluation + +```bash +python scripts/evaluate_external.py --output runs/external-new +``` + +Metrics distinguish: + +- `false_blocks`: valid claims omitted from `allowed_claims`; +- `unsafe_allows`: invalid claims included in `allowed_claims`; +- `release_false_blocks`: valid claims denied by the combination of claim list + and primary verdict (the release gate may disagree with `allowed_claims`); +- `errors`: failures recorded separately, never silently removed or counted as + safe decisions. + +The report includes every case, source/case hashes, independent and observed +means, verdict, blockers, engine-source fingerprint and dependency versions. +Recorded before/after reports are in `results/2026-09-14/`. + +## Run agent alone versus agent with Answerable + +```bash +python scripts/compare_external_agents.py --output runs/agents-preflight +``` + +Without an available adapter and explicit model this writes `status: not_run`, +null metrics and exits 2. It does not fabricate responses or recycle old EMT runs. + +For a real run, configure a **text-only** authenticated model command that reads +a prompt from stdin and writes the model's plain-text answer to stdout. Pass its +argv array with `--command-json` and pin its actual model with `--model`. For +example, with a locally authenticated Claude CLI (replace MODEL_ID with your +actual model identifier): + +```bash +python scripts/compare_external_agents.py \ + --model MODEL_ID \ + --command-json '["claude","-p","--model","MODEL_ID","--output-format","text","--setting-sources","","--disallowedTools","*","--system-prompt","Follow the supplied analytical decision protocol exactly."]' \ + --repetitions 2 --output runs/agents-real +``` + +Authentication/subscription access and the provider's usage charges apply. The +adapter must disable ambient tools and memory; the harness cannot enforce those +properties inside an arbitrary supplied command. Record the exact provider model, +not a moving alias, and verify that the adapter actually uses it. + +The default is 12 cases × 2 repetitions × 2 conditions = 48 decisions. Both arms +get the same aggregate evidence and claim. Neither sees labels or source paths. +Cases are deterministically shuffled and condition order counterbalanced. Every +model invocation starts in a fresh temporary directory without prior conversation. + +The tool arm may answer `CALL_ANSWERABLE`; the harness then executes the real +`assess_answerability` handler, records its response, and asks the model for a +final decision. There is at most one tool call and two model turns per decision. +The model can decline to call the tool and can disagree; tool use is measured, +not assumed. This is an intention-to-treat availability comparison on fixed +summaries, not a native MCP/autonomous-shell benchmark. The second turn is part +of the intervention and adds latency/model usage. + +Prompts, responses, adapter errors and actual tool calls are retained in +`raw.jsonl`; the manifest pins data/question hashes, adapter argv, model, seed and +repetitions. Metrics include false blocks, unsafe allows, tool calls, complete +pairs, improvements, regressions and paired accuracy delta. Timeouts and malformed +responses remain incomplete. Existing run reports cannot be overwritten. + +Unit-test adapters are test doubles only and are never published as agent results. +The recorded preflight currently has **zero real model decisions**. diff --git a/benchmarks/external/manifest.json b/benchmarks/external/manifest.json new file mode 100644 index 0000000..0d58b37 --- /dev/null +++ b/benchmarks/external/manifest.json @@ -0,0 +1,23 @@ +{ + "protocol": "external-smoke-v1", + "retrieved": "2026-09-14", + "normalization": "UTF-8 LF; dates encoded as UTC midnight for day/month labels, not measured instants; no row values changed", + "sources": [ + { + "id": "flights", + "path": "sources/flights.csv", + "url": "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/flights.csv", + "sha256": "237d834127d9c6355630d8f443a7a2377b5925923010009b59809ba0b67f4fac", + "unit": "month", + "units": "thousands of international airline passengers" + }, + { + "id": "temperatures", + "path": "sources/temperatures.csv", + "url": "https://raw.githubusercontent.com/jbrownlee/Datasets/master/daily-min-temperatures.csv", + "sha256": "a31a6961536b0e48a1f8c886d91eb4de50e6747ea5cd5d2d45a8a5039068e406", + "unit": "day", + "units": "degrees Celsius, recorded minimum temperature" + } + ] +} diff --git a/benchmarks/external/results/2026-09-14/README.md b/benchmarks/external/results/2026-09-14/README.md new file mode 100644 index 0000000..10d520f --- /dev/null +++ b/benchmarks/external/results/2026-09-14/README.md @@ -0,0 +1,50 @@ +# Recorded evaluation — 2026-09-14 + +Both engine versions ran the same 12 cases and evaluator in this environment. +The baseline is repository commit `59749d9060c43f0b833a85de1cc3cc3f7e39ddc9`. +The corrected source is identified by `engine_sha256` in `after.json` and the +commit containing this report. `cases_sha256` is identical in both reports. + +| Metric | Before | Corrected | +| --- | ---: | ---: | +| Valid exact descriptive claims | 4 | 4 | +| Invalid claims | 8 | 8 | +| Invalid claims in `allowed_claims` | 8 / 8 | 0 / 8 | +| Valid claims omitted from `allowed_claims` | 0 / 4 | 0 / 4 | +| Valid claims denied by the primary release gate | 2 / 4 | 0 / 4 | +| Execution errors | 0 | 0 | + +The baseline's contradictory verdict/claim outputs are why both claim-level and +release-level metrics are reported. The 8/8 count does not mean every downstream +consumer would publish those claims: a consumer obeying the primary verdict +might reject them despite their appearance in `allowed_claims`. + +This is a small, deliberately targeted regression suite. Datasets come from +outside Answerable; questions and admissibility labels were authored for this +review. Four duplicate cases are artificial perturbations. There is no claim +that these observations estimate general accuracy or were independently judged. + +## Real agent comparison + +**Not run.** The environment had neither Claude/Codex CLI on PATH nor +OpenAI/Anthropic/Gemini API keys. `agent-preflight.json` records zero decisions, +null metrics and the missing adapter. Its manifest describes 48 planned decisions +(12 cases × 2 repetitions × 2 arms); planned decisions are not observations. + +No agent/tool efficacy result is claimed. See the parent suite README for the +command and adapter contract needed to run the paired experiment with a pinned, +authenticated real model. Test doubles only exercise harness plumbing in tests. + +## Reproduce the before/after comparison + +From the corrected checkout, with dependencies installed: + +```bash +python scripts/evaluate_external.py --output runs/external-after-reproduced +git worktree add --detach ../answerable-before 59749d9060c43f0b833a85de1cc3cc3f7e39ddc9 +PYTHONPATH=../answerable-before/src python scripts/evaluate_external.py --output runs/external-before-reproduced +``` + +The baseline command intentionally exits 1 because unsafe allowed claims were +found. The corrected command exits 0. Numerical observed means are also compared +with independently computed means in the automated tests. diff --git a/benchmarks/external/results/2026-09-14/after.json b/benchmarks/external/results/2026-09-14/after.json new file mode 100644 index 0000000..6f1d25d --- /dev/null +++ b/benchmarks/external/results/2026-09-14/after.json @@ -0,0 +1,301 @@ +{ + "protocol": "external-smoke-v1", + "engine_sha256": "a14fd0c8322ba4afde8f472a9a31dbbb70321a2b3ee57d31146e3f65a82e3fb6", + "evaluator_sha256": "ead41b0e799587cfedd0998f0d3564dbd973d288a868e409476349490f4a685b", + "dependencies": { + "duckdb": "1.5.5", + "pyyaml": "6.0.3", + "sqlglot": "29.0.1" + }, + "python": "3.12.14", + "cases_sha256": "eea346e2b4f3a076a9896d4dd40496a7a8c247db82df9a2d21c4229287b60c41", + "metrics": { + "total": 12, + "valid_claims": 4, + "invalid_claims": 8, + "false_blocks": 0, + "unsafe_allows": 0, + "false_block_rate": 0.0, + "unsafe_allow_rate": 0.0, + "release_false_blocks": 0, + "errors": 0 + }, + "limitations": [ + "Two external datasets; questions and labels authored within this project.", + "Duplicate cases are explicit perturbations of external data.", + "No population/causal generalization or independent expert validation.", + "Cases within a source are correlated; no independence-based significance test." + ], + "records": [ + { + "case_id": "flights-full-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "d25114227976050d3e8a3819c32a55033eb5603882b2c3fc203f1f1a30a5e7cd", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "47c5825948cc0df660430dd135fc3cb821d54e36007baacba25ba9c2fe8c67ca", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": false, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "identification_evidence_missing", + "insufficient_power" + ], + "observed_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "b85474b9c087df4021d1903da7cf0b8b540f840a9d337ddd65cf8cf8d5208dc9", + "question_sha256": "2ba86a56e87011c6a6dcfd3d11d122665194d2acab8c24a4f75415d0105341a4", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": false, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 264.13698630136986, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-first_year-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "1e605bef3235d98c40ba3191157f5a7afe4652f810fa8e62471b47f7a4b9e69d", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "2bb5e04df5e99b271b8818dd4547159920fc126349abd035a642a40bff026330", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": false, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "identification_evidence_missing", + "insufficient_power" + ], + "observed_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "149d5d17f183cc498ca3bc1233123464a44ba810c7b1d71de623f086d962f312", + "question_sha256": "f1ee7b62b3f90188a9e3e7c19f72a996dd5f5a4a1d3ec60dea8569c49bef0976", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": false, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 122.71428571428571, + "1": 128.83333333333334 + } + }, + { + "case_id": "temperatures-full-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "c5fc0191c00ea2dd08a731fa0289e53fba9e5bedb80abc64d2d11830b30da4c5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.350275938189856, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-full-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "a3267390b9fdbfe319acc3ffd1646dc2c082097d30e4bcfbd24904e6450e09b5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": false, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "identification_evidence_missing" + ], + "observed_means": { + "0": 12.350275938189856, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-full-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "4293fed050bc156bcd1bcb1d8261737a13f25a0b2d47682dc261c8830fdb9c57", + "question_sha256": "e0ee1de0ad416eec719a4702cc8fc30de14077a4e58f416cf954b32d8877e646", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": false, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 12.35488141202428, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-first_year-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "48a9b6226563afd546b56dfd3ea0fc0970a290e776f0023586cae4371b140a39", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.965193370165743, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "43c4af47404318789d4364c3d5ce4ec3ad3a7663fff19a70d3c2535d65ad4bbd", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": false, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "identification_evidence_missing" + ], + "observed_means": { + "0": 12.965193370165743, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "bd35761ce1641a4b7c8da522a6a5cdef1f98829c5948958a0f5c88743aa17fd3", + "question_sha256": "98de04c22a6d738c93d0acc9212ddc6ab7bcd41611acb7b385d5d4f2a4c8dfeb", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": false, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 13.007692307692302, + "1": 10.092934782608696 + } + } + ] +} diff --git a/benchmarks/external/results/2026-09-14/agent-preflight-manifest.json b/benchmarks/external/results/2026-09-14/agent-preflight-manifest.json new file mode 100644 index 0000000..8ebd057 --- /dev/null +++ b/benchmarks/external/results/2026-09-14/agent-preflight-manifest.json @@ -0,0 +1,168 @@ +{ + "protocol": "paired-external-v1", + "timestamp": "2026-09-14T09:53:42.400792+00:00", + "model": null, + "command": [], + "repetitions": 2, + "seed": 1729, + "case_count": 12, + "cases": [ + { + "case_id": "flights-full-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "acdd508d8a59acb8d09c83713c18b092746a7512cceca2d57aa9a1f53d6087db", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "f005a9b3715e9e2d84e2113c48c6e213c8ce50212c8f45761b664d41e55e78b4", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "b85474b9c087df4021d1903da7cf0b8b540f840a9d337ddd65cf8cf8d5208dc9", + "question_sha256": "5afba688dd58eb460b2237c82ffc8a800f47007e66a676139e3d25b6e8ea2fed", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-first_year-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "363f11f95a0ba4a37bbeb9a959880b96b3dc2dc85a25178c88de0f95e8031600", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "8381c8b4c00b1c51cc7a54d05f164dd6d1bbe9f920dce3879363e6c49b91c730", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "149d5d17f183cc498ca3bc1233123464a44ba810c7b1d71de623f086d962f312", + "question_sha256": "71d5a4bba322d66b42d2765c4520606f782afa8f39ea06b0639b7d33d35b2db0", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "temperatures-full-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "c947ce22978b7e88291bbc82f6b29d725168190166f6b3ac493e9c40cbcb2f4a", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-full-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "b7377ca67c4961984ee67d1b9aeac17f1088f75bcbc8bad413405fbbfa702207", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-full-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "4293fed050bc156bcd1bcb1d8261737a13f25a0b2d47682dc261c8830fdb9c57", + "question_sha256": "1a1d84968ab44c112323f8fe994ee71dcdbd098c0fca1af042914604957b1b46", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-first_year-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "d064e7af03eb962cb0be2a6d725498f783ca161f924674ab3e4212ec1daa3575", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "274a52ddadcb680b76e7cc2f68e461dcc9746e120b3353dff2139ccdb51db661", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "bd35761ce1641a4b7c8da522a6a5cdef1f98829c5948958a0f5c88743aa17fd3", + "question_sha256": "892af0336d8b1f20c82c2a9337577541ab4a4b2b5da70ae7e3b70c15ccb958af", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + } + ], + "adapter_requirement": "text-only, no ambient tools or memory" +} diff --git a/benchmarks/external/results/2026-09-14/agent-preflight.json b/benchmarks/external/results/2026-09-14/agent-preflight.json new file mode 100644 index 0000000..224420b --- /dev/null +++ b/benchmarks/external/results/2026-09-14/agent-preflight.json @@ -0,0 +1,12 @@ +{ + "status": "not_run", + "reason": "No available text-model command and pinned model.", + "detected_cli": { + "claude": null, + "codex": null + }, + "expected_decisions": 48, + "actual_decisions": 0, + "metrics": null, + "manifest_sha256": "004d4e2ece8527fbb0d8d5b1d9153f8398ef19b4022c2853896a5bff12916cb3" +} diff --git a/benchmarks/external/results/2026-09-14/before.json b/benchmarks/external/results/2026-09-14/before.json new file mode 100644 index 0000000..6aada22 --- /dev/null +++ b/benchmarks/external/results/2026-09-14/before.json @@ -0,0 +1,301 @@ +{ + "protocol": "external-smoke-v1", + "engine_sha256": "1d2348457f9cd1c306410b66fc215328d4e12effecb02b5a228791a1fe6e142d", + "evaluator_sha256": "ead41b0e799587cfedd0998f0d3564dbd973d288a868e409476349490f4a685b", + "dependencies": { + "duckdb": "1.5.5", + "pyyaml": "6.0.3", + "sqlglot": "29.0.1" + }, + "python": "3.12.14", + "cases_sha256": "eea346e2b4f3a076a9896d4dd40496a7a8c247db82df9a2d21c4229287b60c41", + "metrics": { + "total": 12, + "valid_claims": 4, + "invalid_claims": 8, + "false_blocks": 0, + "unsafe_allows": 8, + "false_block_rate": 0.0, + "unsafe_allow_rate": 1.0, + "release_false_blocks": 2, + "errors": 0 + }, + "limitations": [ + "Two external datasets; questions and labels authored within this project.", + "Duplicate cases are explicit perturbations of external data.", + "No population/causal generalization or independent expert validation.", + "Cases within a source are correlated; no independence-based significance test." + ], + "records": [ + { + "case_id": "flights-full-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "d25114227976050d3e8a3819c32a55033eb5603882b2c3fc203f1f1a30a5e7cd", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": true, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "insufficient_power" + ], + "observed_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "47c5825948cc0df660430dd135fc3cb821d54e36007baacba25ba9c2fe8c67ca", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": true, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "insufficient_power" + ], + "observed_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "b85474b9c087df4021d1903da7cf0b8b540f840a9d337ddd65cf8cf8d5208dc9", + "question_sha256": "2ba86a56e87011c6a6dcfd3d11d122665194d2acab8c24a4f75415d0105341a4", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + }, + "allowed": true, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities", + "insufficient_power" + ], + "observed_means": { + "0": 264.13698630136986, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-first_year-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "1e605bef3235d98c40ba3191157f5a7afe4652f810fa8e62471b47f7a4b9e69d", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": true, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "insufficient_power" + ], + "observed_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "2bb5e04df5e99b271b8818dd4547159920fc126349abd035a642a40bff026330", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": true, + "verdict": "NOT_ANSWERABLE_YET", + "release_allowed": false, + "blockers": [ + "insufficient_power" + ], + "observed_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "149d5d17f183cc498ca3bc1233123464a44ba810c7b1d71de623f086d962f312", + "question_sha256": "f1ee7b62b3f90188a9e3e7c19f72a996dd5f5a4a1d3ec60dea8569c49bef0976", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + }, + "allowed": true, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities", + "insufficient_power" + ], + "observed_means": { + "0": 122.71428571428571, + "1": 128.83333333333334 + } + }, + { + "case_id": "temperatures-full-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "c5fc0191c00ea2dd08a731fa0289e53fba9e5bedb80abc64d2d11830b30da4c5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.350275938189856, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-full-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "a3267390b9fdbfe319acc3ffd1646dc2c082097d30e4bcfbd24904e6450e09b5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.350275938189856, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-full-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "4293fed050bc156bcd1bcb1d8261737a13f25a0b2d47682dc261c8830fdb9c57", + "question_sha256": "e0ee1de0ad416eec719a4702cc8fc30de14077a4e58f416cf954b32d8877e646", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + }, + "allowed": true, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 12.35488141202428, + "1": 10.021817192600656 + } + }, + { + "case_id": "temperatures-first_year-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "48a9b6226563afd546b56dfd3ea0fc0970a290e776f0023586cae4371b140a39", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.965193370165743, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "43c4af47404318789d4364c3d5ce4ec3ad3a7663fff19a70d3c2535d65ad4bbd", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": true, + "verdict": "ANSWERABLE", + "release_allowed": true, + "blockers": [], + "observed_means": { + "0": 12.965193370165743, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "bd35761ce1641a4b7c8da522a6a5cdef1f98829c5948958a0f5c88743aa17fd3", + "question_sha256": "98de04c22a6d738c93d0acc9212ddc6ab7bcd41611acb7b385d5d4f2a4c8dfeb", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + }, + "allowed": true, + "verdict": "DATA_INTEGRITY_FAILURE", + "release_allowed": false, + "blockers": [ + "duplicate_entities" + ], + "observed_means": { + "0": 13.007692307692302, + "1": 10.092934782608696 + } + } + ] +} diff --git a/benchmarks/external/results/2026-09-14/cases.json b/benchmarks/external/results/2026-09-14/cases.json new file mode 100644 index 0000000..e93b212 --- /dev/null +++ b/benchmarks/external/results/2026-09-14/cases.json @@ -0,0 +1,158 @@ +[ + { + "case_id": "flights-full-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "d25114227976050d3e8a3819c32a55033eb5603882b2c3fc203f1f1a30a5e7cd", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "bc393bf899eca2b67062c2eaaba6cb798146e61990f41816fb22bfe75df59f56", + "question_sha256": "47c5825948cc0df660430dd135fc3cb821d54e36007baacba25ba9c2fe8c67ca", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-full-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 266.25 in January-June and 294.347 in July-December (thousands of international airline passengers).", + "data_sha256": "b85474b9c087df4021d1903da7cf0b8b540f840a9d337ddd65cf8cf8d5208dc9", + "question_sha256": "2ba86a56e87011c6a6dcfd3d11d122665194d2acab8c24a4f75415d0105341a4", + "expected_means": { + "0": 266.25, + "1": 294.34722222222223 + } + }, + { + "case_id": "flights-first_year-observed", + "source": "flights", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "1e605bef3235d98c40ba3191157f5a7afe4652f810fa8e62471b47f7a4b9e69d", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-causal", + "source": "flights", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "4e6f8d715d5f2b3e19293a01a8712cb93dadfb76575d4ed31095a245be4f1e78", + "question_sha256": "2bb5e04df5e99b271b8818dd4547159920fc126349abd035a642a40bff026330", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "flights-first_year-duplicate", + "source": "flights", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded months, the observed mean was 124.5 in January-June and 128.833 in July-December (thousands of international airline passengers).", + "data_sha256": "149d5d17f183cc498ca3bc1233123464a44ba810c7b1d71de623f086d962f312", + "question_sha256": "f1ee7b62b3f90188a9e3e7c19f72a996dd5f5a4a1d3ec60dea8569c49bef0976", + "expected_means": { + "0": 124.5, + "1": 128.83333333333334 + } + }, + { + "case_id": "temperatures-full-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "c5fc0191c00ea2dd08a731fa0289e53fba9e5bedb80abc64d2d11830b30da4c5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-full-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3e55cafa171cdaf69b50efe6c5cecc1698c0ee1c087659219113565ec8ea855d", + "question_sha256": "a3267390b9fdbfe319acc3ffd1646dc2c082097d30e4bcfbd24904e6450e09b5", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-full-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.3503 in January-June and 10.0218 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "4293fed050bc156bcd1bcb1d8261737a13f25a0b2d47682dc261c8830fdb9c57", + "question_sha256": "e0ee1de0ad416eec719a4702cc8fc30de14077a4e58f416cf954b32d8877e646", + "expected_means": { + "0": 12.350275938189846, + "1": 10.021817192600652 + } + }, + { + "case_id": "temperatures-first_year-observed", + "source": "temperatures", + "variant": "observed", + "expected_allow": true, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "48a9b6226563afd546b56dfd3ea0fc0970a290e776f0023586cae4371b140a39", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-causal", + "source": "temperatures", + "variant": "causal", + "expected_allow": false, + "claim": "Assignment to July-December caused the observed difference in outcomes.", + "data_sha256": "3eae6469bbb4043aef429ee6f63f57019fe7abec9fbbef6a3f59a2ca2f3d60a7", + "question_sha256": "43c4af47404318789d4364c3d5ce4ec3ad3a7663fff19a70d3c2535d65ad4bbd", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + }, + { + "case_id": "temperatures-first_year-duplicate", + "source": "temperatures", + "variant": "duplicate", + "expected_allow": false, + "claim": "Among these recorded days, the observed mean was 12.9652 in January-June and 10.0929 in July-December (degrees Celsius, recorded minimum temperature).", + "data_sha256": "bd35761ce1641a4b7c8da522a6a5cdef1f98829c5948958a0f5c88743aa17fd3", + "question_sha256": "98de04c22a6d738c93d0acc9212ddc6ab7bcd41611acb7b385d5d4f2a4c8dfeb", + "expected_means": { + "0": 12.965193370165746, + "1": 10.092934782608696 + } + } +] diff --git a/benchmarks/external/results/2026-09-14/verification.txt b/benchmarks/external/results/2026-09-14/verification.txt new file mode 100644 index 0000000..2adaa9b --- /dev/null +++ b/benchmarks/external/results/2026-09-14/verification.txt @@ -0,0 +1,14 @@ +Local verification, 2026-09-14 + +make verify: PASS +215 tests passed; branch-aware coverage 96.01% (required 95%). +Ruff formatting/lint: PASS +Strict mypy: PASS (73 source files) +Public schemas: PASS (21) +Traceability: PASS (119 requirements) +make build: PASS (sdist and wheel) + +Six initial adversarial tests were executed before implementation and all failed. +The corrected targeted suite (adversarial + external/harness) passed 23 tests. +Tests using fake model responses validate harness plumbing only. +No real model decisions were produced. diff --git a/benchmarks/external/sources/flights.csv b/benchmarks/external/sources/flights.csv new file mode 100644 index 0000000..831265b --- /dev/null +++ b/benchmarks/external/sources/flights.csv @@ -0,0 +1,145 @@ +year,month,passengers +1949,January,112 +1949,February,118 +1949,March,132 +1949,April,129 +1949,May,121 +1949,June,135 +1949,July,148 +1949,August,148 +1949,September,136 +1949,October,119 +1949,November,104 +1949,December,118 +1950,January,115 +1950,February,126 +1950,March,141 +1950,April,135 +1950,May,125 +1950,June,149 +1950,July,170 +1950,August,170 +1950,September,158 +1950,October,133 +1950,November,114 +1950,December,140 +1951,January,145 +1951,February,150 +1951,March,178 +1951,April,163 +1951,May,172 +1951,June,178 +1951,July,199 +1951,August,199 +1951,September,184 +1951,October,162 +1951,November,146 +1951,December,166 +1952,January,171 +1952,February,180 +1952,March,193 +1952,April,181 +1952,May,183 +1952,June,218 +1952,July,230 +1952,August,242 +1952,September,209 +1952,October,191 +1952,November,172 +1952,December,194 +1953,January,196 +1953,February,196 +1953,March,236 +1953,April,235 +1953,May,229 +1953,June,243 +1953,July,264 +1953,August,272 +1953,September,237 +1953,October,211 +1953,November,180 +1953,December,201 +1954,January,204 +1954,February,188 +1954,March,235 +1954,April,227 +1954,May,234 +1954,June,264 +1954,July,302 +1954,August,293 +1954,September,259 +1954,October,229 +1954,November,203 +1954,December,229 +1955,January,242 +1955,February,233 +1955,March,267 +1955,April,269 +1955,May,270 +1955,June,315 +1955,July,364 +1955,August,347 +1955,September,312 +1955,October,274 +1955,November,237 +1955,December,278 +1956,January,284 +1956,February,277 +1956,March,317 +1956,April,313 +1956,May,318 +1956,June,374 +1956,July,413 +1956,August,405 +1956,September,355 +1956,October,306 +1956,November,271 +1956,December,306 +1957,January,315 +1957,February,301 +1957,March,356 +1957,April,348 +1957,May,355 +1957,June,422 +1957,July,465 +1957,August,467 +1957,September,404 +1957,October,347 +1957,November,305 +1957,December,336 +1958,January,340 +1958,February,318 +1958,March,362 +1958,April,348 +1958,May,363 +1958,June,435 +1958,July,491 +1958,August,505 +1958,September,404 +1958,October,359 +1958,November,310 +1958,December,337 +1959,January,360 +1959,February,342 +1959,March,406 +1959,April,396 +1959,May,420 +1959,June,472 +1959,July,548 +1959,August,559 +1959,September,463 +1959,October,407 +1959,November,362 +1959,December,405 +1960,January,417 +1960,February,391 +1960,March,419 +1960,April,461 +1960,May,472 +1960,June,535 +1960,July,622 +1960,August,606 +1960,September,508 +1960,October,461 +1960,November,390 +1960,December,432 diff --git a/benchmarks/external/sources/temperatures.csv b/benchmarks/external/sources/temperatures.csv new file mode 100644 index 0000000..75e835a --- /dev/null +++ b/benchmarks/external/sources/temperatures.csv @@ -0,0 +1,3651 @@ +"Date","Temp" +"1981-01-01",20.7 +"1981-01-02",17.9 +"1981-01-03",18.8 +"1981-01-04",14.6 +"1981-01-05",15.8 +"1981-01-06",15.8 +"1981-01-07",15.8 +"1981-01-08",17.4 +"1981-01-09",21.8 +"1981-01-10",20.0 +"1981-01-11",16.2 +"1981-01-12",13.3 +"1981-01-13",16.7 +"1981-01-14",21.5 +"1981-01-15",25.0 +"1981-01-16",20.7 +"1981-01-17",20.6 +"1981-01-18",24.8 +"1981-01-19",17.7 +"1981-01-20",15.5 +"1981-01-21",18.2 +"1981-01-22",12.1 +"1981-01-23",14.4 +"1981-01-24",16.0 +"1981-01-25",16.5 +"1981-01-26",18.7 +"1981-01-27",19.4 +"1981-01-28",17.2 +"1981-01-29",15.5 +"1981-01-30",15.1 +"1981-01-31",15.4 +"1981-02-01",15.3 +"1981-02-02",18.8 +"1981-02-03",21.9 +"1981-02-04",19.9 +"1981-02-05",16.6 +"1981-02-06",16.8 +"1981-02-07",14.6 +"1981-02-08",17.1 +"1981-02-09",25.0 +"1981-02-10",15.0 +"1981-02-11",13.7 +"1981-02-12",13.9 +"1981-02-13",18.3 +"1981-02-14",22.0 +"1981-02-15",22.1 +"1981-02-16",21.2 +"1981-02-17",18.4 +"1981-02-18",16.6 +"1981-02-19",16.1 +"1981-02-20",15.7 +"1981-02-21",16.6 +"1981-02-22",16.5 +"1981-02-23",14.4 +"1981-02-24",14.4 +"1981-02-25",18.5 +"1981-02-26",16.9 +"1981-02-27",17.5 +"1981-02-28",21.2 +"1981-03-01",17.8 +"1981-03-02",18.6 +"1981-03-03",17.0 +"1981-03-04",16.0 +"1981-03-05",13.3 +"1981-03-06",14.3 +"1981-03-07",11.4 +"1981-03-08",16.3 +"1981-03-09",16.1 +"1981-03-10",11.8 +"1981-03-11",12.2 +"1981-03-12",14.7 +"1981-03-13",11.8 +"1981-03-14",11.3 +"1981-03-15",10.6 +"1981-03-16",11.7 +"1981-03-17",14.2 +"1981-03-18",11.2 +"1981-03-19",16.9 +"1981-03-20",16.7 +"1981-03-21",8.1 +"1981-03-22",8.0 +"1981-03-23",8.8 +"1981-03-24",13.4 +"1981-03-25",10.9 +"1981-03-26",13.4 +"1981-03-27",11.0 +"1981-03-28",15.0 +"1981-03-29",15.7 +"1981-03-30",14.5 +"1981-03-31",15.8 +"1981-04-01",16.7 +"1981-04-02",16.8 +"1981-04-03",17.5 +"1981-04-04",17.1 +"1981-04-05",18.1 +"1981-04-06",16.6 +"1981-04-07",10.0 +"1981-04-08",14.9 +"1981-04-09",15.9 +"1981-04-10",13.0 +"1981-04-11",7.6 +"1981-04-12",11.5 +"1981-04-13",13.5 +"1981-04-14",13.0 +"1981-04-15",13.3 +"1981-04-16",12.1 +"1981-04-17",12.4 +"1981-04-18",13.2 +"1981-04-19",13.8 +"1981-04-20",10.6 +"1981-04-21",9.0 +"1981-04-22",10.0 +"1981-04-23",9.8 +"1981-04-24",11.5 +"1981-04-25",8.9 +"1981-04-26",7.4 +"1981-04-27",9.9 +"1981-04-28",9.3 +"1981-04-29",9.9 +"1981-04-30",7.4 +"1981-05-01",8.6 +"1981-05-02",11.9 +"1981-05-03",14.0 +"1981-05-04",8.6 +"1981-05-05",10.0 +"1981-05-06",13.5 +"1981-05-07",12.0 +"1981-05-08",10.5 +"1981-05-09",10.7 +"1981-05-10",8.1 +"1981-05-11",10.1 +"1981-05-12",10.6 +"1981-05-13",5.3 +"1981-05-14",6.6 +"1981-05-15",8.5 +"1981-05-16",11.2 +"1981-05-17",9.8 +"1981-05-18",5.9 +"1981-05-19",3.2 +"1981-05-20",2.1 +"1981-05-21",3.4 +"1981-05-22",5.4 +"1981-05-23",9.6 +"1981-05-24",11.5 +"1981-05-25",12.3 +"1981-05-26",12.6 +"1981-05-27",11.0 +"1981-05-28",11.2 +"1981-05-29",11.4 +"1981-05-30",11.8 +"1981-05-31",12.8 +"1981-06-01",11.6 +"1981-06-02",10.6 +"1981-06-03",9.8 +"1981-06-04",11.2 +"1981-06-05",5.7 +"1981-06-06",7.1 +"1981-06-07",2.5 +"1981-06-08",3.5 +"1981-06-09",4.6 +"1981-06-10",11.0 +"1981-06-11",5.7 +"1981-06-12",7.7 +"1981-06-13",10.4 +"1981-06-14",11.4 +"1981-06-15",9.2 +"1981-06-16",6.1 +"1981-06-17",2.7 +"1981-06-18",4.3 +"1981-06-19",6.3 +"1981-06-20",3.8 +"1981-06-21",4.4 +"1981-06-22",7.1 +"1981-06-23",4.8 +"1981-06-24",5.8 +"1981-06-25",6.2 +"1981-06-26",7.3 +"1981-06-27",9.2 +"1981-06-28",10.2 +"1981-06-29",9.5 +"1981-06-30",9.5 +"1981-07-01",10.7 +"1981-07-02",10.0 +"1981-07-03",6.5 +"1981-07-04",7.0 +"1981-07-05",7.4 +"1981-07-06",8.1 +"1981-07-07",6.6 +"1981-07-08",8.3 +"1981-07-09",8.9 +"1981-07-10",4.6 +"1981-07-11",6.8 +"1981-07-12",5.7 +"1981-07-13",6.1 +"1981-07-14",7.0 +"1981-07-15",7.2 +"1981-07-16",6.3 +"1981-07-17",8.8 +"1981-07-18",5.0 +"1981-07-19",7.4 +"1981-07-20",10.1 +"1981-07-21",12.0 +"1981-07-22",9.0 +"1981-07-23",8.9 +"1981-07-24",9.8 +"1981-07-25",9.0 +"1981-07-26",9.2 +"1981-07-27",7.7 +"1981-07-28",8.0 +"1981-07-29",6.1 +"1981-07-30",3.5 +"1981-07-31",3.2 +"1981-08-01",5.7 +"1981-08-02",7.7 +"1981-08-03",9.0 +"1981-08-04",10.0 +"1981-08-05",6.2 +"1981-08-06",6.9 +"1981-08-07",6.5 +"1981-08-08",6.8 +"1981-08-09",7.0 +"1981-08-10",5.2 +"1981-08-11",3.0 +"1981-08-12",5.6 +"1981-08-13",7.9 +"1981-08-14",9.0 +"1981-08-15",8.6 +"1981-08-16",10.3 +"1981-08-17",10.5 +"1981-08-18",7.6 +"1981-08-19",9.7 +"1981-08-20",12.5 +"1981-08-21",7.4 +"1981-08-22",7.9 +"1981-08-23",3.9 +"1981-08-24",6.6 +"1981-08-25",4.6 +"1981-08-26",7.0 +"1981-08-27",6.0 +"1981-08-28",5.5 +"1981-08-29",8.1 +"1981-08-30",5.5 +"1981-08-31",6.2 +"1981-09-01",8.0 +"1981-09-02",10.3 +"1981-09-03",9.8 +"1981-09-04",9.6 +"1981-09-05",8.5 +"1981-09-06",7.5 +"1981-09-07",11.2 +"1981-09-08",14.6 +"1981-09-09",11.7 +"1981-09-10",7.8 +"1981-09-11",12.3 +"1981-09-12",10.1 +"1981-09-13",11.5 +"1981-09-14",7.3 +"1981-09-15",10.9 +"1981-09-16",14.1 +"1981-09-17",10.7 +"1981-09-18",16.9 +"1981-09-19",10.5 +"1981-09-20",6.5 +"1981-09-21",11.0 +"1981-09-22",6.3 +"1981-09-23",10.5 +"1981-09-24",7.2 +"1981-09-25",7.6 +"1981-09-26",10.7 +"1981-09-27",7.8 +"1981-09-28",9.6 +"1981-09-29",11.4 +"1981-09-30",12.4 +"1981-10-01",8.9 +"1981-10-02",13.2 +"1981-10-03",8.6 +"1981-10-04",6.2 +"1981-10-05",11.4 +"1981-10-06",13.2 +"1981-10-07",14.3 +"1981-10-08",7.3 +"1981-10-09",12.9 +"1981-10-10",7.8 +"1981-10-11",6.2 +"1981-10-12",5.6 +"1981-10-13",10.0 +"1981-10-14",13.3 +"1981-10-15",8.3 +"1981-10-16",10.2 +"1981-10-17",8.6 +"1981-10-18",7.3 +"1981-10-19",10.4 +"1981-10-20",11.2 +"1981-10-21",13.2 +"1981-10-22",11.4 +"1981-10-23",9.1 +"1981-10-24",6.6 +"1981-10-25",8.4 +"1981-10-26",9.7 +"1981-10-27",13.2 +"1981-10-28",12.5 +"1981-10-29",11.0 +"1981-10-30",11.0 +"1981-10-31",11.7 +"1981-11-01",9.2 +"1981-11-02",11.5 +"1981-11-03",13.6 +"1981-11-04",13.7 +"1981-11-05",10.4 +"1981-11-06",11.5 +"1981-11-07",7.6 +"1981-11-08",9.6 +"1981-11-09",14.2 +"1981-11-10",15.7 +"1981-11-11",10.5 +"1981-11-12",10.5 +"1981-11-13",9.7 +"1981-11-14",9.5 +"1981-11-15",11.3 +"1981-11-16",8.9 +"1981-11-17",9.4 +"1981-11-18",11.9 +"1981-11-19",11.7 +"1981-11-20",13.4 +"1981-11-21",12.6 +"1981-11-22",10.1 +"1981-11-23",15.8 +"1981-11-24",13.6 +"1981-11-25",11.9 +"1981-11-26",9.9 +"1981-11-27",12.6 +"1981-11-28",17.8 +"1981-11-29",15.0 +"1981-11-30",13.6 +"1981-12-01",13.4 +"1981-12-02",10.5 +"1981-12-03",14.2 +"1981-12-04",11.5 +"1981-12-05",13.0 +"1981-12-06",15.0 +"1981-12-07",14.7 +"1981-12-08",12.6 +"1981-12-09",12.5 +"1981-12-10",13.5 +"1981-12-11",14.8 +"1981-12-12",17.2 +"1981-12-13",9.7 +"1981-12-14",12.1 +"1981-12-15",12.8 +"1981-12-16",11.2 +"1981-12-17",16.4 +"1981-12-18",15.6 +"1981-12-19",13.3 +"1981-12-20",11.0 +"1981-12-21",11.1 +"1981-12-22",15.0 +"1981-12-23",12.8 +"1981-12-24",15.0 +"1981-12-25",14.2 +"1981-12-26",14.0 +"1981-12-27",15.5 +"1981-12-28",13.3 +"1981-12-29",15.6 +"1981-12-30",15.2 +"1981-12-31",17.4 +"1982-01-01",17.0 +"1982-01-02",15.0 +"1982-01-03",13.5 +"1982-01-04",15.2 +"1982-01-05",13.0 +"1982-01-06",12.5 +"1982-01-07",14.1 +"1982-01-08",14.8 +"1982-01-09",16.2 +"1982-01-10",15.8 +"1982-01-11",19.1 +"1982-01-12",22.2 +"1982-01-13",15.9 +"1982-01-14",13.0 +"1982-01-15",14.1 +"1982-01-16",15.8 +"1982-01-17",24.0 +"1982-01-18",18.0 +"1982-01-19",19.7 +"1982-01-20",25.2 +"1982-01-21",20.5 +"1982-01-22",19.3 +"1982-01-23",15.8 +"1982-01-24",17.0 +"1982-01-25",18.4 +"1982-01-26",13.3 +"1982-01-27",14.6 +"1982-01-28",12.5 +"1982-01-29",17.0 +"1982-01-30",17.1 +"1982-01-31",14.0 +"1982-02-01",14.6 +"1982-02-02",13.3 +"1982-02-03",14.8 +"1982-02-04",15.1 +"1982-02-05",13.1 +"1982-02-06",13.6 +"1982-02-07",19.5 +"1982-02-08",22.7 +"1982-02-09",17.2 +"1982-02-10",13.5 +"1982-02-11",15.4 +"1982-02-12",17.0 +"1982-02-13",19.2 +"1982-02-14",22.8 +"1982-02-15",26.3 +"1982-02-16",18.2 +"1982-02-17",17.0 +"1982-02-18",14.8 +"1982-02-19",12.8 +"1982-02-20",15.5 +"1982-02-21",15.6 +"1982-02-22",13.1 +"1982-02-23",15.2 +"1982-02-24",14.1 +"1982-02-25",12.5 +"1982-02-26",14.6 +"1982-02-27",10.4 +"1982-02-28",13.9 +"1982-03-01",11.9 +"1982-03-02",13.5 +"1982-03-03",9.8 +"1982-03-04",14.0 +"1982-03-05",21.5 +"1982-03-06",19.5 +"1982-03-07",16.7 +"1982-03-08",19.1 +"1982-03-09",11.0 +"1982-03-10",9.0 +"1982-03-11",10.0 +"1982-03-12",14.6 +"1982-03-13",12.5 +"1982-03-14",17.2 +"1982-03-15",19.2 +"1982-03-16",22.2 +"1982-03-17",15.7 +"1982-03-18",14.2 +"1982-03-19",9.8 +"1982-03-20",14.0 +"1982-03-21",17.5 +"1982-03-22",20.7 +"1982-03-23",15.6 +"1982-03-24",13.2 +"1982-03-25",14.5 +"1982-03-26",16.8 +"1982-03-27",17.2 +"1982-03-28",13.4 +"1982-03-29",14.2 +"1982-03-30",14.3 +"1982-03-31",10.2 +"1982-04-01",10.4 +"1982-04-02",12.3 +"1982-04-03",11.9 +"1982-04-04",11.2 +"1982-04-05",8.5 +"1982-04-06",12.0 +"1982-04-07",12.4 +"1982-04-08",12.9 +"1982-04-09",10.1 +"1982-04-10",15.0 +"1982-04-11",13.6 +"1982-04-12",12.4 +"1982-04-13",13.6 +"1982-04-14",16.1 +"1982-04-15",19.5 +"1982-04-16",14.2 +"1982-04-17",9.3 +"1982-04-18",10.1 +"1982-04-19",7.4 +"1982-04-20",8.6 +"1982-04-21",7.8 +"1982-04-22",9.1 +"1982-04-23",13.0 +"1982-04-24",16.5 +"1982-04-25",12.9 +"1982-04-26",6.9 +"1982-04-27",6.9 +"1982-04-28",8.7 +"1982-04-29",10.0 +"1982-04-30",10.8 +"1982-05-01",7.5 +"1982-05-02",6.3 +"1982-05-03",11.9 +"1982-05-04",13.8 +"1982-05-05",11.8 +"1982-05-06",11.0 +"1982-05-07",10.1 +"1982-05-08",8.5 +"1982-05-09",5.5 +"1982-05-10",7.6 +"1982-05-11",8.7 +"1982-05-12",10.8 +"1982-05-13",11.2 +"1982-05-14",9.1 +"1982-05-15",3.7 +"1982-05-16",4.6 +"1982-05-17",6.6 +"1982-05-18",13.2 +"1982-05-19",15.2 +"1982-05-20",7.6 +"1982-05-21",8.4 +"1982-05-22",6.0 +"1982-05-23",8.3 +"1982-05-24",8.6 +"1982-05-25",11.1 +"1982-05-26",12.1 +"1982-05-27",12.9 +"1982-05-28",14.0 +"1982-05-29",12.5 +"1982-05-30",11.5 +"1982-05-31",7.0 +"1982-06-01",7.1 +"1982-06-02",9.0 +"1982-06-03",3.1 +"1982-06-04",2.5 +"1982-06-05",0.0 +"1982-06-06",1.6 +"1982-06-07",2.6 +"1982-06-08",5.7 +"1982-06-09",2.3 +"1982-06-10",4.5 +"1982-06-11",8.2 +"1982-06-12",6.9 +"1982-06-13",7.3 +"1982-06-14",6.0 +"1982-06-15",7.3 +"1982-06-16",7.6 +"1982-06-17",8.0 +"1982-06-18",8.0 +"1982-06-19",6.8 +"1982-06-20",7.3 +"1982-06-21",6.2 +"1982-06-22",6.9 +"1982-06-23",8.9 +"1982-06-24",4.0 +"1982-06-25",1.3 +"1982-06-26",0.8 +"1982-06-27",4.3 +"1982-06-28",7.3 +"1982-06-29",7.7 +"1982-06-30",9.0 +"1982-07-01",4.2 +"1982-07-02",1.6 +"1982-07-03",2.6 +"1982-07-04",3.4 +"1982-07-05",3.9 +"1982-07-06",7.0 +"1982-07-07",7.8 +"1982-07-08",5.3 +"1982-07-09",2.4 +"1982-07-10",2.8 +"1982-07-11",4.0 +"1982-07-12",7.5 +"1982-07-13",7.8 +"1982-07-14",5.6 +"1982-07-15",3.3 +"1982-07-16",5.0 +"1982-07-17",3.7 +"1982-07-18",3.9 +"1982-07-19",5.2 +"1982-07-20",0.2 +"1982-07-21",0.8 +"1982-07-22",0.9 +"1982-07-23",3.5 +"1982-07-24",6.6 +"1982-07-25",9.5 +"1982-07-26",9.0 +"1982-07-27",3.5 +"1982-07-28",4.5 +"1982-07-29",5.7 +"1982-07-30",5.6 +"1982-07-31",7.1 +"1982-08-01",9.7 +"1982-08-02",8.3 +"1982-08-03",9.1 +"1982-08-04",2.8 +"1982-08-05",2.2 +"1982-08-06",4.5 +"1982-08-07",3.8 +"1982-08-08",3.8 +"1982-08-09",6.2 +"1982-08-10",11.5 +"1982-08-11",10.2 +"1982-08-12",7.9 +"1982-08-13",9.0 +"1982-08-14",9.5 +"1982-08-15",6.0 +"1982-08-16",8.2 +"1982-08-17",9.2 +"1982-08-18",4.3 +"1982-08-19",6.6 +"1982-08-20",9.4 +"1982-08-21",13.2 +"1982-08-22",6.6 +"1982-08-23",5.1 +"1982-08-24",12.1 +"1982-08-25",11.2 +"1982-08-26",8.5 +"1982-08-27",4.6 +"1982-08-28",7.0 +"1982-08-29",14.2 +"1982-08-30",12.7 +"1982-08-31",7.6 +"1982-09-01",4.0 +"1982-09-02",10.0 +"1982-09-03",10.5 +"1982-09-04",5.0 +"1982-09-05",4.5 +"1982-09-06",8.2 +"1982-09-07",4.3 +"1982-09-08",9.8 +"1982-09-09",5.8 +"1982-09-10",5.0 +"1982-09-11",8.5 +"1982-09-12",9.0 +"1982-09-13",3.6 +"1982-09-14",6.7 +"1982-09-15",6.7 +"1982-09-16",10.1 +"1982-09-17",15.0 +"1982-09-18",8.9 +"1982-09-19",5.7 +"1982-09-20",4.2 +"1982-09-21",4.0 +"1982-09-22",5.3 +"1982-09-23",6.3 +"1982-09-24",8.5 +"1982-09-25",11.5 +"1982-09-26",7.7 +"1982-09-27",9.2 +"1982-09-28",7.8 +"1982-09-29",6.3 +"1982-09-30",6.3 +"1982-10-01",8.6 +"1982-10-02",6.1 +"1982-10-03",13.2 +"1982-10-04",9.9 +"1982-10-05",4.7 +"1982-10-06",5.8 +"1982-10-07",14.9 +"1982-10-08",10.7 +"1982-10-09",8.6 +"1982-10-10",9.4 +"1982-10-11",5.7 +"1982-10-12",10.9 +"1982-10-13",13.1 +"1982-10-14",10.4 +"1982-10-15",8.2 +"1982-10-16",9.8 +"1982-10-17",7.5 +"1982-10-18",5.8 +"1982-10-19",9.8 +"1982-10-20",7.9 +"1982-10-21",8.7 +"1982-10-22",10.0 +"1982-10-23",10.6 +"1982-10-24",8.0 +"1982-10-25",10.2 +"1982-10-26",15.1 +"1982-10-27",13.9 +"1982-10-28",9.2 +"1982-10-29",9.0 +"1982-10-30",13.2 +"1982-10-31",7.0 +"1982-11-01",10.6 +"1982-11-02",6.9 +"1982-11-03",9.5 +"1982-11-04",12.5 +"1982-11-05",13.6 +"1982-11-06",17.7 +"1982-11-07",16.0 +"1982-11-08",11.3 +"1982-11-09",10.5 +"1982-11-10",14.4 +"1982-11-11",10.3 +"1982-11-12",9.0 +"1982-11-13",11.1 +"1982-11-14",14.5 +"1982-11-15",18.0 +"1982-11-16",12.8 +"1982-11-17",10.7 +"1982-11-18",9.1 +"1982-11-19",8.7 +"1982-11-20",12.4 +"1982-11-21",12.6 +"1982-11-22",10.3 +"1982-11-23",13.7 +"1982-11-24",16.0 +"1982-11-25",15.8 +"1982-11-26",12.1 +"1982-11-27",12.5 +"1982-11-28",12.2 +"1982-11-29",13.7 +"1982-11-30",16.1 +"1982-12-01",15.5 +"1982-12-02",10.3 +"1982-12-03",10.5 +"1982-12-04",11.0 +"1982-12-05",11.9 +"1982-12-06",13.0 +"1982-12-07",12.2 +"1982-12-08",10.6 +"1982-12-09",13.0 +"1982-12-10",13.0 +"1982-12-11",12.2 +"1982-12-12",12.6 +"1982-12-13",18.7 +"1982-12-14",15.2 +"1982-12-15",15.3 +"1982-12-16",13.9 +"1982-12-17",15.8 +"1982-12-18",13.0 +"1982-12-19",13.0 +"1982-12-20",13.7 +"1982-12-21",12.0 +"1982-12-22",10.8 +"1982-12-23",15.6 +"1982-12-24",15.3 +"1982-12-25",13.9 +"1982-12-26",13.0 +"1982-12-27",15.3 +"1982-12-28",16.3 +"1982-12-29",15.8 +"1982-12-30",17.7 +"1982-12-31",16.3 +"1983-01-01",18.4 +"1983-01-02",15.0 +"1983-01-03",10.9 +"1983-01-04",11.4 +"1983-01-05",14.8 +"1983-01-06",12.1 +"1983-01-07",12.8 +"1983-01-08",16.2 +"1983-01-09",15.5 +"1983-01-10",13.0 +"1983-01-11",10.5 +"1983-01-12",9.1 +"1983-01-13",10.5 +"1983-01-14",11.8 +"1983-01-15",12.7 +"1983-01-16",12.7 +"1983-01-17",11.5 +"1983-01-18",13.8 +"1983-01-19",13.3 +"1983-01-20",11.6 +"1983-01-21",15.4 +"1983-01-22",12.4 +"1983-01-23",16.9 +"1983-01-24",14.7 +"1983-01-25",10.6 +"1983-01-26",15.6 +"1983-01-27",10.7 +"1983-01-28",12.6 +"1983-01-29",13.8 +"1983-01-30",14.3 +"1983-01-31",14.0 +"1983-02-01",18.1 +"1983-02-02",17.3 +"1983-02-03",13.0 +"1983-02-04",16.0 +"1983-02-05",14.9 +"1983-02-06",16.2 +"1983-02-07",20.3 +"1983-02-08",22.5 +"1983-02-09",17.2 +"1983-02-10",15.9 +"1983-02-11",16.8 +"1983-02-12",13.8 +"1983-02-13",12.8 +"1983-02-14",14.0 +"1983-02-15",17.5 +"1983-02-16",21.5 +"1983-02-17",16.8 +"1983-02-18",13.6 +"1983-02-19",14.5 +"1983-02-20",14.2 +"1983-02-21",15.7 +"1983-02-22",19.7 +"1983-02-23",17.4 +"1983-02-24",14.4 +"1983-02-25",16.9 +"1983-02-26",19.1 +"1983-02-27",20.4 +"1983-02-28",20.1 +"1983-03-01",19.9 +"1983-03-02",22.0 +"1983-03-03",20.5 +"1983-03-04",22.1 +"1983-03-05",20.6 +"1983-03-06",15.0 +"1983-03-07",20.6 +"1983-03-08",21.5 +"1983-03-09",16.2 +"1983-03-10",14.1 +"1983-03-11",14.5 +"1983-03-12",21.1 +"1983-03-13",15.9 +"1983-03-14",15.2 +"1983-03-15",13.1 +"1983-03-16",13.2 +"1983-03-17",12.5 +"1983-03-18",15.2 +"1983-03-19",17.6 +"1983-03-20",15.5 +"1983-03-21",16.7 +"1983-03-22",16.3 +"1983-03-23",15.1 +"1983-03-24",12.7 +"1983-03-25",10.0 +"1983-03-26",11.4 +"1983-03-27",12.6 +"1983-03-28",10.7 +"1983-03-29",10.0 +"1983-03-30",13.9 +"1983-03-31",13.4 +"1983-04-01",12.5 +"1983-04-02",12.8 +"1983-04-03",7.8 +"1983-04-04",11.1 +"1983-04-05",10.7 +"1983-04-06",7.1 +"1983-04-07",6.7 +"1983-04-08",5.7 +"1983-04-09",9.1 +"1983-04-10",15.2 +"1983-04-11",15.5 +"1983-04-12",11.1 +"1983-04-13",11.7 +"1983-04-14",11.5 +"1983-04-15",9.8 +"1983-04-16",6.2 +"1983-04-17",6.7 +"1983-04-18",7.5 +"1983-04-19",8.8 +"1983-04-20",8.0 +"1983-04-21",10.4 +"1983-04-22",14.5 +"1983-04-23",16.5 +"1983-04-24",14.1 +"1983-04-25",10.5 +"1983-04-26",12.6 +"1983-04-27",13.0 +"1983-04-28",8.7 +"1983-04-29",10.1 +"1983-04-30",12.0 +"1983-05-01",12.5 +"1983-05-02",13.5 +"1983-05-03",13.7 +"1983-05-04",13.5 +"1983-05-05",10.7 +"1983-05-06",13.0 +"1983-05-07",11.6 +"1983-05-08",13.0 +"1983-05-09",11.2 +"1983-05-10",13.5 +"1983-05-11",12.9 +"1983-05-12",6.8 +"1983-05-13",10.0 +"1983-05-14",14.5 +"1983-05-15",11.7 +"1983-05-16",6.7 +"1983-05-17",4.6 +"1983-05-18",4.9 +"1983-05-19",7.4 +"1983-05-20",8.3 +"1983-05-21",7.5 +"1983-05-22",6.2 +"1983-05-23",7.8 +"1983-05-24",13.2 +"1983-05-25",11.9 +"1983-05-26",6.5 +"1983-05-27",8.3 +"1983-05-28",12.1 +"1983-05-29",9.3 +"1983-05-30",7.5 +"1983-05-31",9.3 +"1983-06-01",11.0 +"1983-06-02",10.8 +"1983-06-03",5.3 +"1983-06-04",7.6 +"1983-06-05",5.6 +"1983-06-06",7.2 +"1983-06-07",9.6 +"1983-06-08",7.0 +"1983-06-09",8.3 +"1983-06-10",7.8 +"1983-06-11",4.7 +"1983-06-12",6.8 +"1983-06-13",7.2 +"1983-06-14",8.3 +"1983-06-15",9.5 +"1983-06-16",4.7 +"1983-06-17",3.0 +"1983-06-18",1.5 +"1983-06-19",2.5 +"1983-06-20",6.2 +"1983-06-21",11.6 +"1983-06-22",6.6 +"1983-06-23",6.6 +"1983-06-24",8.0 +"1983-06-25",7.9 +"1983-06-26",3.3 +"1983-06-27",3.9 +"1983-06-28",6.0 +"1983-06-29",4.0 +"1983-06-30",5.5 +"1983-07-01",8.5 +"1983-07-02",9.8 +"1983-07-03",9.5 +"1983-07-04",7.2 +"1983-07-05",8.1 +"1983-07-06",8.0 +"1983-07-07",8.5 +"1983-07-08",8.8 +"1983-07-09",8.3 +"1983-07-10",2.4 +"1983-07-11",4.9 +"1983-07-12",5.9 +"1983-07-13",6.7 +"1983-07-14",8.4 +"1983-07-15",6.5 +"1983-07-16",7.9 +"1983-07-17",4.1 +"1983-07-18",5.4 +"1983-07-19",7.5 +"1983-07-20",3.9 +"1983-07-21",2.5 +"1983-07-22",5.3 +"1983-07-23",6.6 +"1983-07-24",0.0 +"1983-07-25",0.7 +"1983-07-26",7.6 +"1983-07-27",12.3 +"1983-07-28",9.2 +"1983-07-29",9.6 +"1983-07-30",9.5 +"1983-07-31",10.0 +"1983-08-01",7.7 +"1983-08-02",8.0 +"1983-08-03",8.3 +"1983-08-04",8.3 +"1983-08-05",4.5 +"1983-08-06",6.5 +"1983-08-07",9.4 +"1983-08-08",9.4 +"1983-08-09",10.5 +"1983-08-10",10.7 +"1983-08-11",9.9 +"1983-08-12",7.6 +"1983-08-13",5.8 +"1983-08-14",8.5 +"1983-08-15",13.8 +"1983-08-16",14.3 +"1983-08-17",8.3 +"1983-08-18",5.3 +"1983-08-19",3.0 +"1983-08-20",5.2 +"1983-08-21",10.3 +"1983-08-22",11.1 +"1983-08-23",10.5 +"1983-08-24",9.0 +"1983-08-25",13.0 +"1983-08-26",6.4 +"1983-08-27",8.4 +"1983-08-28",6.7 +"1983-08-29",8.3 +"1983-08-30",11.2 +"1983-08-31",10.0 +"1983-09-01",10.1 +"1983-09-02",10.6 +"1983-09-03",10.9 +"1983-09-04",5.7 +"1983-09-05",9.5 +"1983-09-06",10.4 +"1983-09-07",11.1 +"1983-09-08",12.2 +"1983-09-09",10.6 +"1983-09-10",8.8 +"1983-09-11",9.2 +"1983-09-12",5.5 +"1983-09-13",7.1 +"1983-09-14",6.5 +"1983-09-15",4.3 +"1983-09-16",5.0 +"1983-09-17",11.2 +"1983-09-18",7.5 +"1983-09-19",12.0 +"1983-09-20",13.6 +"1983-09-21",8.3 +"1983-09-22",8.5 +"1983-09-23",12.9 +"1983-09-24",7.7 +"1983-09-25",7.6 +"1983-09-26",3.5 +"1983-09-27",10.4 +"1983-09-28",15.4 +"1983-09-29",10.6 +"1983-09-30",9.6 +"1983-10-01",9.3 +"1983-10-02",13.9 +"1983-10-03",7.7 +"1983-10-04",9.5 +"1983-10-05",7.6 +"1983-10-06",6.9 +"1983-10-07",6.8 +"1983-10-08",5.8 +"1983-10-09",6.0 +"1983-10-10",8.3 +"1983-10-11",9.1 +"1983-10-12",12.5 +"1983-10-13",13.2 +"1983-10-14",16.2 +"1983-10-15",12.5 +"1983-10-16",11.8 +"1983-10-17",10.6 +"1983-10-18",10.0 +"1983-10-19",12.2 +"1983-10-20",8.9 +"1983-10-21",10.3 +"1983-10-22",7.5 +"1983-10-23",11.6 +"1983-10-24",12.6 +"1983-10-25",12.9 +"1983-10-26",11.7 +"1983-10-27",14.0 +"1983-10-28",12.3 +"1983-10-29",9.0 +"1983-10-30",9.2 +"1983-10-31",9.8 +"1983-11-01",11.8 +"1983-11-02",10.6 +"1983-11-03",12.6 +"1983-11-04",11.0 +"1983-11-05",8.2 +"1983-11-06",7.5 +"1983-11-07",13.6 +"1983-11-08",14.8 +"1983-11-09",10.9 +"1983-11-10",7.7 +"1983-11-11",10.2 +"1983-11-12",10.8 +"1983-11-13",10.8 +"1983-11-14",12.5 +"1983-11-15",13.2 +"1983-11-16",8.7 +"1983-11-17",5.7 +"1983-11-18",9.8 +"1983-11-19",7.3 +"1983-11-20",10.8 +"1983-11-21",10.0 +"1983-11-22",16.2 +"1983-11-23",15.0 +"1983-11-24",14.5 +"1983-11-25",15.9 +"1983-11-26",14.9 +"1983-11-27",14.2 +"1983-11-28",15.8 +"1983-11-29",17.2 +"1983-11-30",17.6 +"1983-12-01",12.1 +"1983-12-02",11.4 +"1983-12-03",13.0 +"1983-12-04",13.2 +"1983-12-05",12.0 +"1983-12-06",15.3 +"1983-12-07",12.7 +"1983-12-08",12.1 +"1983-12-09",13.8 +"1983-12-10",10.9 +"1983-12-11",12.0 +"1983-12-12",16.5 +"1983-12-13",15.0 +"1983-12-14",11.2 +"1983-12-15",13.9 +"1983-12-16",15.0 +"1983-12-17",14.8 +"1983-12-18",15.0 +"1983-12-19",13.3 +"1983-12-20",20.4 +"1983-12-21",18.0 +"1983-12-22",12.2 +"1983-12-23",16.7 +"1983-12-24",13.8 +"1983-12-25",17.5 +"1983-12-26",15.0 +"1983-12-27",13.9 +"1983-12-28",11.1 +"1983-12-29",16.1 +"1983-12-30",20.4 +"1983-12-31",18.0 +"1984-01-01",19.5 +"1984-01-02",17.1 +"1984-01-03",17.1 +"1984-01-04",12.0 +"1984-01-05",11.0 +"1984-01-06",16.3 +"1984-01-07",16.1 +"1984-01-08",13.0 +"1984-01-09",13.4 +"1984-01-10",15.2 +"1984-01-11",12.5 +"1984-01-12",14.3 +"1984-01-13",16.5 +"1984-01-14",18.6 +"1984-01-15",18.0 +"1984-01-16",18.2 +"1984-01-17",11.4 +"1984-01-18",11.9 +"1984-01-19",12.2 +"1984-01-20",14.8 +"1984-01-21",13.1 +"1984-01-22",12.7 +"1984-01-23",10.5 +"1984-01-24",13.8 +"1984-01-25",18.8 +"1984-01-26",13.9 +"1984-01-27",11.2 +"1984-01-28",10.6 +"1984-01-29",14.7 +"1984-01-30",13.1 +"1984-01-31",12.1 +"1984-02-01",14.7 +"1984-02-02",11.1 +"1984-02-03",13.0 +"1984-02-04",15.6 +"1984-02-05",14.2 +"1984-02-06",15.5 +"1984-02-07",18.0 +"1984-02-08",15.0 +"1984-02-09",15.9 +"1984-02-10",15.5 +"1984-02-11",15.8 +"1984-02-12",16.6 +"1984-02-13",13.6 +"1984-02-14",13.8 +"1984-02-15",14.6 +"1984-02-16",15.6 +"1984-02-17",16.6 +"1984-02-18",14.3 +"1984-02-19",16.3 +"1984-02-20",18.9 +"1984-02-21",18.7 +"1984-02-22",14.5 +"1984-02-23",16.5 +"1984-02-24",14.1 +"1984-02-25",13.5 +"1984-02-26",11.7 +"1984-02-27",15.1 +"1984-02-28",11.2 +"1984-02-29",13.5 +"1984-03-01",12.6 +"1984-03-02",8.8 +"1984-03-03",10.5 +"1984-03-04",12.1 +"1984-03-05",14.5 +"1984-03-06",19.5 +"1984-03-07",14.0 +"1984-03-08",13.8 +"1984-03-09",10.5 +"1984-03-10",13.8 +"1984-03-11",11.4 +"1984-03-12",15.6 +"1984-03-13",11.1 +"1984-03-14",12.1 +"1984-03-15",14.2 +"1984-03-16",10.9 +"1984-03-17",14.2 +"1984-03-18",13.8 +"1984-03-19",15.1 +"1984-03-20",14.0 +"1984-03-21",12.1 +"1984-03-22",13.8 +"1984-03-23",16.6 +"1984-03-24",17.8 +"1984-03-25",9.4 +"1984-03-26",10.2 +"1984-03-27",7.4 +"1984-03-28",8.7 +"1984-03-29",14.0 +"1984-03-30",15.3 +"1984-03-31",11.1 +"1984-04-01",9.7 +"1984-04-02",10.3 +"1984-04-03",9.2 +"1984-04-04",8.2 +"1984-04-05",9.7 +"1984-04-06",12.4 +"1984-04-07",12.5 +"1984-04-08",9.0 +"1984-04-09",9.7 +"1984-04-10",10.1 +"1984-04-11",11.2 +"1984-04-12",12.0 +"1984-04-13",11.1 +"1984-04-14",10.8 +"1984-04-15",12.8 +"1984-04-16",9.8 +"1984-04-17",13.7 +"1984-04-18",11.0 +"1984-04-19",13.2 +"1984-04-20",13.0 +"1984-04-21",10.2 +"1984-04-22",13.2 +"1984-04-23",9.3 +"1984-04-24",11.1 +"1984-04-25",10.3 +"1984-04-26",8.7 +"1984-04-27",11.7 +"1984-04-28",12.5 +"1984-04-29",6.5 +"1984-04-30",9.6 +"1984-05-01",13.8 +"1984-05-02",14.7 +"1984-05-03",9.1 +"1984-05-04",4.8 +"1984-05-05",3.3 +"1984-05-06",3.5 +"1984-05-07",5.7 +"1984-05-08",5.5 +"1984-05-09",7.0 +"1984-05-10",9.5 +"1984-05-11",9.9 +"1984-05-12",4.9 +"1984-05-13",6.3 +"1984-05-14",4.8 +"1984-05-15",6.2 +"1984-05-16",7.1 +"1984-05-17",7.5 +"1984-05-18",9.4 +"1984-05-19",8.7 +"1984-05-20",9.5 +"1984-05-21",12.1 +"1984-05-22",9.5 +"1984-05-23",9.3 +"1984-05-24",8.5 +"1984-05-25",8.0 +"1984-05-26",9.8 +"1984-05-27",6.2 +"1984-05-28",7.3 +"1984-05-29",10.9 +"1984-05-30",10.0 +"1984-05-31",8.7 +"1984-06-01",9.0 +"1984-06-02",10.8 +"1984-06-03",12.4 +"1984-06-04",7.2 +"1984-06-05",7.2 +"1984-06-06",11.1 +"1984-06-07",9.3 +"1984-06-08",10.1 +"1984-06-09",3.9 +"1984-06-10",5.0 +"1984-06-11",8.2 +"1984-06-12",2.8 +"1984-06-13",4.3 +"1984-06-14",8.1 +"1984-06-15",11.1 +"1984-06-16",4.7 +"1984-06-17",5.3 +"1984-06-18",10.0 +"1984-06-19",5.6 +"1984-06-20",2.2 +"1984-06-21",7.1 +"1984-06-22",8.3 +"1984-06-23",8.6 +"1984-06-24",10.1 +"1984-06-25",8.3 +"1984-06-26",7.2 +"1984-06-27",7.7 +"1984-06-28",7.8 +"1984-06-29",9.1 +"1984-06-30",9.4 +"1984-07-01",7.8 +"1984-07-02",2.6 +"1984-07-03",2.4 +"1984-07-04",3.9 +"1984-07-05",1.3 +"1984-07-06",2.1 +"1984-07-07",7.4 +"1984-07-08",7.2 +"1984-07-09",8.8 +"1984-07-10",8.9 +"1984-07-11",8.8 +"1984-07-12",8.0 +"1984-07-13",0.7 +"1984-07-14",0.1 +"1984-07-15",0.9 +"1984-07-16",7.8 +"1984-07-17",7.2 +"1984-07-18",8.0 +"1984-07-19",4.6 +"1984-07-20",5.2 +"1984-07-21",5.8 +"1984-07-22",6.8 +"1984-07-23",8.1 +"1984-07-24",7.5 +"1984-07-25",5.4 +"1984-07-26",4.6 +"1984-07-27",6.4 +"1984-07-28",9.7 +"1984-07-29",7.0 +"1984-07-30",10.0 +"1984-07-31",10.6 +"1984-08-01",11.5 +"1984-08-02",10.2 +"1984-08-03",11.1 +"1984-08-04",11.0 +"1984-08-05",8.9 +"1984-08-06",9.9 +"1984-08-07",11.7 +"1984-08-08",11.6 +"1984-08-09",9.0 +"1984-08-10",6.3 +"1984-08-11",8.7 +"1984-08-12",8.5 +"1984-08-13",8.5 +"1984-08-14",8.0 +"1984-08-15",6.0 +"1984-08-16",8.0 +"1984-08-17",8.5 +"1984-08-18",7.7 +"1984-08-19",8.4 +"1984-08-20",9.0 +"1984-08-21",8.3 +"1984-08-22",6.8 +"1984-08-23",9.3 +"1984-08-24",6.7 +"1984-08-25",9.0 +"1984-08-26",7.3 +"1984-08-27",6.3 +"1984-08-28",7.9 +"1984-08-29",5.2 +"1984-08-30",9.0 +"1984-08-31",11.3 +"1984-09-01",9.2 +"1984-09-02",11.3 +"1984-09-03",7.0 +"1984-09-04",8.0 +"1984-09-05",4.6 +"1984-09-06",8.5 +"1984-09-07",9.5 +"1984-09-08",9.4 +"1984-09-09",10.5 +"1984-09-10",9.7 +"1984-09-11",4.9 +"1984-09-12",8.0 +"1984-09-13",5.8 +"1984-09-14",5.5 +"1984-09-15",10.9 +"1984-09-16",11.7 +"1984-09-17",9.2 +"1984-09-18",8.9 +"1984-09-19",11.3 +"1984-09-20",8.6 +"1984-09-21",6.2 +"1984-09-22",6.6 +"1984-09-23",9.1 +"1984-09-24",6.1 +"1984-09-25",7.5 +"1984-09-26",10.7 +"1984-09-27",6.3 +"1984-09-28",5.5 +"1984-09-29",6.7 +"1984-09-30",4.2 +"1984-10-01",11.3 +"1984-10-02",16.3 +"1984-10-03",10.5 +"1984-10-04",10.3 +"1984-10-05",7.9 +"1984-10-06",7.7 +"1984-10-07",16.0 +"1984-10-08",14.6 +"1984-10-09",12.5 +"1984-10-10",8.1 +"1984-10-11",12.2 +"1984-10-12",17.2 +"1984-10-13",9.4 +"1984-10-14",8.7 +"1984-10-15",5.9 +"1984-10-16",4.8 +"1984-10-17",7.4 +"1984-10-18",9.4 +"1984-10-19",9.7 +"1984-10-20",9.9 +"1984-10-21",6.5 +"1984-10-22",9.8 +"1984-10-23",18.2 +"1984-10-24",11.3 +"1984-10-25",9.1 +"1984-10-26",9.6 +"1984-10-27",13.5 +"1984-10-28",10.7 +"1984-10-29",10.0 +"1984-10-30",8.5 +"1984-10-31",12.6 +"1984-11-01",16.6 +"1984-11-02",11.6 +"1984-11-03",12.2 +"1984-11-04",11.2 +"1984-11-05",9.2 +"1984-11-06",9.9 +"1984-11-07",11.9 +"1984-11-08",15.6 +"1984-11-09",19.0 +"1984-11-10",12.8 +"1984-11-11",12.2 +"1984-11-12",12.0 +"1984-11-13",11.1 +"1984-11-14",11.8 +"1984-11-15",7.6 +"1984-11-16",13.0 +"1984-11-17",12.7 +"1984-11-18",16.0 +"1984-11-19",14.8 +"1984-11-20",14.2 +"1984-11-21",10.0 +"1984-11-22",8.8 +"1984-11-23",11.6 +"1984-11-24",8.6 +"1984-11-25",14.6 +"1984-11-26",24.3 +"1984-11-27",11.6 +"1984-11-28",10.8 +"1984-11-29",12.0 +"1984-11-30",11.0 +"1984-12-01",12.6 +"1984-12-02",10.8 +"1984-12-03",9.1 +"1984-12-04",11.0 +"1984-12-05",13.0 +"1984-12-06",12.8 +"1984-12-07",9.9 +"1984-12-08",11.6 +"1984-12-09",10.5 +"1984-12-10",15.9 +"1984-12-11",12.2 +"1984-12-12",13.0 +"1984-12-13",12.5 +"1984-12-14",12.5 +"1984-12-15",11.4 +"1984-12-16",12.1 +"1984-12-17",16.8 +"1984-12-18",12.1 +"1984-12-19",11.3 +"1984-12-20",10.4 +"1984-12-21",14.2 +"1984-12-22",11.4 +"1984-12-23",13.7 +"1984-12-24",16.5 +"1984-12-25",12.8 +"1984-12-26",12.2 +"1984-12-27",12.0 +"1984-12-28",12.6 +"1984-12-29",16.0 +"1984-12-30",16.4 +"1985-01-01",13.3 +"1985-01-02",15.2 +"1985-01-03",13.1 +"1985-01-04",12.7 +"1985-01-05",14.6 +"1985-01-06",11.0 +"1985-01-07",13.2 +"1985-01-08",12.2 +"1985-01-09",14.4 +"1985-01-10",13.7 +"1985-01-11",14.5 +"1985-01-12",14.1 +"1985-01-13",14.4 +"1985-01-14",19.7 +"1985-01-15",16.5 +"1985-01-16",15.9 +"1985-01-17",11.8 +"1985-01-18",12.0 +"1985-01-19",11.4 +"1985-01-20",14.4 +"1985-01-21",12.4 +"1985-01-22",15.1 +"1985-01-23",15.6 +"1985-01-24",15.2 +"1985-01-25",12.8 +"1985-01-26",13.3 +"1985-01-27",17.5 +"1985-01-28",15.4 +"1985-01-29",13.5 +"1985-01-30",16.7 +"1985-01-31",15.2 +"1985-02-01",14.9 +"1985-02-02",10.2 +"1985-02-03",13.6 +"1985-02-04",19.0 +"1985-02-05",15.7 +"1985-02-06",18.0 +"1985-02-07",14.8 +"1985-02-08",13.9 +"1985-02-09",13.0 +"1985-02-10",15.3 +"1985-02-11",14.3 +"1985-02-12",15.6 +"1985-02-13",16.0 +"1985-02-14",14.9 +"1985-02-15",11.1 +"1985-02-16",14.8 +"1985-02-17",13.0 +"1985-02-18",12.2 +"1985-02-19",10.9 +"1985-02-20",14.6 +"1985-02-21",16.6 +"1985-02-22",18.1 +"1985-02-23",13.4 +"1985-02-24",10.3 +"1985-02-25",13.6 +"1985-02-26",13.8 +"1985-02-27",10.3 +"1985-02-28",11.0 +"1985-03-01",14.3 +"1985-03-02",15.5 +"1985-03-03",14.7 +"1985-03-04",12.7 +"1985-03-05",10.7 +"1985-03-06",12.6 +"1985-03-07",9.8 +"1985-03-08",13.2 +"1985-03-09",15.2 +"1985-03-10",16.6 +"1985-03-11",21.0 +"1985-03-12",22.4 +"1985-03-13",17.0 +"1985-03-14",21.7 +"1985-03-15",21.4 +"1985-03-16",18.6 +"1985-03-17",16.2 +"1985-03-18",16.8 +"1985-03-19",17.0 +"1985-03-20",18.4 +"1985-03-21",17.2 +"1985-03-22",18.4 +"1985-03-23",18.8 +"1985-03-24",16.5 +"1985-03-25",13.3 +"1985-03-26",12.2 +"1985-03-27",11.3 +"1985-03-28",13.8 +"1985-03-29",16.6 +"1985-03-30",14.0 +"1985-03-31",14.3 +"1985-04-01",16.4 +"1985-04-02",11.9 +"1985-04-03",15.7 +"1985-04-04",17.6 +"1985-04-05",17.5 +"1985-04-06",15.9 +"1985-04-07",16.2 +"1985-04-08",16.0 +"1985-04-09",15.9 +"1985-04-10",16.2 +"1985-04-11",16.2 +"1985-04-12",19.5 +"1985-04-13",18.2 +"1985-04-14",21.8 +"1985-04-15",15.1 +"1985-04-16",11.0 +"1985-04-17",8.1 +"1985-04-18",9.5 +"1985-04-19",9.3 +"1985-04-20",10.6 +"1985-04-21",6.3 +"1985-04-22",8.6 +"1985-04-23",6.8 +"1985-04-24",8.7 +"1985-04-25",8.4 +"1985-04-26",9.3 +"1985-04-27",10.0 +"1985-04-28",10.5 +"1985-04-29",12.0 +"1985-04-30",10.1 +"1985-05-01",9.4 +"1985-05-02",10.1 +"1985-05-03",8.0 +"1985-05-04",10.6 +"1985-05-05",13.6 +"1985-05-06",15.4 +"1985-05-07",9.0 +"1985-05-08",10.4 +"1985-05-09",11.0 +"1985-05-10",12.1 +"1985-05-11",13.4 +"1985-05-12",11.3 +"1985-05-13",6.7 +"1985-05-14",9.8 +"1985-05-15",10.8 +"1985-05-16",7.8 +"1985-05-17",4.5 +"1985-05-18",7.6 +"1985-05-19",6.9 +"1985-05-20",7.5 +"1985-05-21",8.5 +"1985-05-22",5.5 +"1985-05-23",9.5 +"1985-05-24",7.3 +"1985-05-25",5.4 +"1985-05-26",5.5 +"1985-05-27",8.1 +"1985-05-28",11.2 +"1985-05-29",13.4 +"1985-05-30",11.6 +"1985-05-31",10.1 +"1985-06-01",4.3 +"1985-06-02",5.5 +"1985-06-03",4.4 +"1985-06-04",5.9 +"1985-06-05",5.7 +"1985-06-06",8.2 +"1985-06-07",8.2 +"1985-06-08",4.2 +"1985-06-09",6.5 +"1985-06-10",10.0 +"1985-06-11",8.8 +"1985-06-12",6.6 +"1985-06-13",7.8 +"1985-06-14",10.1 +"1985-06-15",7.1 +"1985-06-16",7.7 +"1985-06-17",8.5 +"1985-06-18",7.3 +"1985-06-19",6.9 +"1985-06-20",8.4 +"1985-06-21",7.1 +"1985-06-22",6.3 +"1985-06-23",0.6 +"1985-06-24",1.6 +"1985-06-25",7.0 +"1985-06-26",8.3 +"1985-06-27",8.0 +"1985-06-28",10.2 +"1985-06-29",10.6 +"1985-06-30",10.4 +"1985-07-01",11.6 +"1985-07-02",11.0 +"1985-07-03",10.7 +"1985-07-04",7.3 +"1985-07-05",4.2 +"1985-07-06",4.7 +"1985-07-07",5.6 +"1985-07-08",7.7 +"1985-07-09",7.5 +"1985-07-10",4.9 +"1985-07-11",5.9 +"1985-07-12",7.8 +"1985-07-13",5.8 +"1985-07-14",7.0 +"1985-07-15",8.4 +"1985-07-16",6.2 +"1985-07-17",7.5 +"1985-07-18",4.8 +"1985-07-19",3.3 +"1985-07-20",3.2 +"1985-07-21",7.0 +"1985-07-22",8.4 +"1985-07-23",0.3 +"1985-07-24",0.3 +"1985-07-25",2.1 +"1985-07-26",8.5 +"1985-07-27",1.4 +"1985-07-28",4.1 +"1985-07-29",10.3 +"1985-07-30",6.6 +"1985-07-31",6.1 +"1985-08-01",7.0 +"1985-08-02",5.1 +"1985-08-03",6.3 +"1985-08-04",6.9 +"1985-08-05",11.4 +"1985-08-06",10.4 +"1985-08-07",10.3 +"1985-08-08",9.2 +"1985-08-09",7.2 +"1985-08-10",7.5 +"1985-08-11",4.0 +"1985-08-12",5.6 +"1985-08-13",6.7 +"1985-08-14",8.4 +"1985-08-15",11.0 +"1985-08-16",8.4 +"1985-08-17",8.8 +"1985-08-18",8.6 +"1985-08-19",8.3 +"1985-08-20",4.0 +"1985-08-21",3.6 +"1985-08-22",5.7 +"1985-08-23",10.6 +"1985-08-24",6.9 +"1985-08-25",10.0 +"1985-08-26",9.8 +"1985-08-27",7.2 +"1985-08-28",10.5 +"1985-08-29",3.6 +"1985-08-30",5.3 +"1985-08-31",8.4 +"1985-09-01",10.3 +"1985-09-02",7.9 +"1985-09-03",8.5 +"1985-09-04",7.9 +"1985-09-05",8.0 +"1985-09-06",9.8 +"1985-09-07",6.7 +"1985-09-08",4.8 +"1985-09-09",9.9 +"1985-09-10",12.8 +"1985-09-11",10.9 +"1985-09-12",11.7 +"1985-09-13",11.7 +"1985-09-14",11.0 +"1985-09-15",8.2 +"1985-09-16",7.5 +"1985-09-17",5.4 +"1985-09-18",7.2 +"1985-09-19",9.7 +"1985-09-20",8.4 +"1985-09-21",9.0 +"1985-09-22",8.7 +"1985-09-23",6.6 +"1985-09-24",11.6 +"1985-09-25",13.1 +"1985-09-26",6.7 +"1985-09-27",6.5 +"1985-09-28",7.7 +"1985-09-29",8.7 +"1985-09-30",7.2 +"1985-10-01",10.5 +"1985-10-02",8.6 +"1985-10-03",7.2 +"1985-10-04",11.4 +"1985-10-05",16.2 +"1985-10-06",6.1 +"1985-10-07",9.6 +"1985-10-08",11.1 +"1985-10-09",13.6 +"1985-10-10",10.7 +"1985-10-11",14.7 +"1985-10-12",11.6 +"1985-10-13",7.3 +"1985-10-14",8.0 +"1985-10-15",9.6 +"1985-10-16",16.0 +"1985-10-17",15.1 +"1985-10-18",12.8 +"1985-10-19",6.2 +"1985-10-20",7.1 +"1985-10-21",8.4 +"1985-10-22",10.0 +"1985-10-23",12.7 +"1985-10-24",10.0 +"1985-10-25",10.2 +"1985-10-26",6.5 +"1985-10-27",9.2 +"1985-10-28",11.9 +"1985-10-29",14.7 +"1985-10-30",11.4 +"1985-10-31",6.8 +"1985-11-01",7.4 +"1985-11-02",11.2 +"1985-11-03",9.2 +"1985-11-04",12.6 +"1985-11-05",16.0 +"1985-11-06",17.1 +"1985-11-07",15.3 +"1985-11-08",13.3 +"1985-11-09",15.4 +"1985-11-10",13.2 +"1985-11-11",14.4 +"1985-11-12",14.0 +"1985-11-13",15.5 +"1985-11-14",21.0 +"1985-11-15",10.0 +"1985-11-16",9.6 +"1985-11-17",12.0 +"1985-11-18",12.2 +"1985-11-19",11.3 +"1985-11-20",13.2 +"1985-11-21",10.5 +"1985-11-22",10.1 +"1985-11-23",8.8 +"1985-11-24",13.7 +"1985-11-25",16.2 +"1985-11-26",16.0 +"1985-11-27",14.0 +"1985-11-28",13.7 +"1985-11-29",12.5 +"1985-11-30",12.8 +"1985-12-01",12.3 +"1985-12-02",15.2 +"1985-12-03",15.0 +"1985-12-04",16.4 +"1985-12-05",16.1 +"1985-12-06",14.6 +"1985-12-07",18.2 +"1985-12-08",16.4 +"1985-12-09",16.6 +"1985-12-10",14.7 +"1985-12-11",15.8 +"1985-12-12",14.1 +"1985-12-13",13.5 +"1985-12-14",13.6 +"1985-12-15",13.7 +"1985-12-16",13.6 +"1985-12-17",12.1 +"1985-12-18",12.7 +"1985-12-19",13.3 +"1985-12-20",14.2 +"1985-12-21",15.0 +"1985-12-22",13.7 +"1985-12-23",12.0 +"1985-12-24",13.1 +"1985-12-25",13.2 +"1985-12-26",13.3 +"1985-12-27",11.5 +"1985-12-28",10.8 +"1985-12-29",12.0 +"1985-12-30",16.3 +"1985-12-31",14.4 +"1986-01-01",12.9 +"1986-01-02",13.8 +"1986-01-03",10.6 +"1986-01-04",12.6 +"1986-01-05",13.7 +"1986-01-06",12.6 +"1986-01-07",13.1 +"1986-01-08",15.4 +"1986-01-09",11.9 +"1986-01-10",13.8 +"1986-01-11",14.4 +"1986-01-12",15.2 +"1986-01-13",12.5 +"1986-01-14",12.2 +"1986-01-15",16.1 +"1986-01-16",14.6 +"1986-01-17",11.6 +"1986-01-18",13.1 +"1986-01-19",12.8 +"1986-01-20",15.2 +"1986-01-21",13.8 +"1986-01-22",15.0 +"1986-01-23",13.5 +"1986-01-24",11.8 +"1986-01-25",15.3 +"1986-01-26",13.5 +"1986-01-27",15.3 +"1986-01-28",13.8 +"1986-01-29",15.8 +"1986-01-30",17.4 +"1986-01-31",15.3 +"1986-02-01",14.6 +"1986-02-02",14.8 +"1986-02-03",10.7 +"1986-02-04",11.6 +"1986-02-05",13.6 +"1986-02-06",14.4 +"1986-02-07",11.8 +"1986-02-08",15.8 +"1986-02-09",16.0 +"1986-02-10",11.8 +"1986-02-11",14.5 +"1986-02-12",10.7 +"1986-02-13",14.2 +"1986-02-14",19.5 +"1986-02-15",21.4 +"1986-02-16",17.9 +"1986-02-17",17.4 +"1986-02-18",12.7 +"1986-02-19",13.8 +"1986-02-20",14.0 +"1986-02-21",15.0 +"1986-02-22",14.5 +"1986-02-23",13.1 +"1986-02-24",11.4 +"1986-02-25",12.5 +"1986-02-26",12.0 +"1986-02-27",13.4 +"1986-02-28",14.4 +"1986-03-01",17.7 +"1986-03-02",13.9 +"1986-03-03",13.3 +"1986-03-04",14.6 +"1986-03-05",16.4 +"1986-03-06",16.8 +"1986-03-07",20.0 +"1986-03-08",12.5 +"1986-03-09",12.7 +"1986-03-10",11.7 +"1986-03-11",12.7 +"1986-03-12",8.6 +"1986-03-13",11.9 +"1986-03-14",16.0 +"1986-03-15",15.2 +"1986-03-16",13.4 +"1986-03-17",11.6 +"1986-03-18",11.1 +"1986-03-19",15.6 +"1986-03-20",17.0 +"1986-03-21",18.5 +"1986-03-22",17.4 +"1986-03-23",16.5 +"1986-03-24",16.2 +"1986-03-25",16.1 +"1986-03-26",13.2 +"1986-03-27",18.0 +"1986-03-28",12.8 +"1986-03-29",11.7 +"1986-03-30",16.7 +"1986-03-31",15.6 +"1986-04-01",10.2 +"1986-04-02",10.3 +"1986-04-03",15.0 +"1986-04-04",18.0 +"1986-04-05",13.8 +"1986-04-06",10.5 +"1986-04-07",11.8 +"1986-04-08",7.2 +"1986-04-09",11.6 +"1986-04-10",7.4 +"1986-04-11",14.2 +"1986-04-12",12.2 +"1986-04-13",9.0 +"1986-04-14",12.3 +"1986-04-15",19.7 +"1986-04-16",12.8 +"1986-04-17",12.4 +"1986-04-18",12.0 +"1986-04-19",12.0 +"1986-04-20",11.1 +"1986-04-21",12.7 +"1986-04-22",14.2 +"1986-04-23",11.6 +"1986-04-24",12.0 +"1986-04-25",11.5 +"1986-04-26",8.3 +"1986-04-27",10.5 +"1986-04-28",9.0 +"1986-04-29",6.9 +"1986-04-30",9.4 +"1986-05-01",11.1 +"1986-05-02",9.1 +"1986-05-03",7.7 +"1986-05-04",10.0 +"1986-05-05",10.4 +"1986-05-06",8.0 +"1986-05-07",9.8 +"1986-05-08",12.4 +"1986-05-09",12.9 +"1986-05-10",12.3 +"1986-05-11",6.9 +"1986-05-12",10.5 +"1986-05-13",11.0 +"1986-05-14",9.7 +"1986-05-15",11.1 +"1986-05-16",11.5 +"1986-05-17",13.4 +"1986-05-18",10.9 +"1986-05-19",12.0 +"1986-05-20",12.1 +"1986-05-21",10.4 +"1986-05-22",10.0 +"1986-05-23",9.6 +"1986-05-24",11.3 +"1986-05-25",8.5 +"1986-05-26",6.3 +"1986-05-27",8.2 +"1986-05-28",10.7 +"1986-05-29",10.3 +"1986-05-30",9.5 +"1986-05-31",10.9 +"1986-06-01",10.9 +"1986-06-02",4.3 +"1986-06-03",5.2 +"1986-06-04",11.0 +"1986-06-05",11.6 +"1986-06-06",10.6 +"1986-06-07",9.4 +"1986-06-08",10.0 +"1986-06-09",9.6 +"1986-06-10",9.5 +"1986-06-11",9.7 +"1986-06-12",9.6 +"1986-06-13",7.0 +"1986-06-14",7.0 +"1986-06-15",6.8 +"1986-06-16",6.9 +"1986-06-17",8.0 +"1986-06-18",7.6 +"1986-06-19",8.6 +"1986-06-20",5.7 +"1986-06-21",5.5 +"1986-06-22",5.7 +"1986-06-23",5.7 +"1986-06-24",6.6 +"1986-06-25",6.0 +"1986-06-26",6.9 +"1986-06-27",7.7 +"1986-06-28",8.0 +"1986-06-29",3.9 +"1986-06-30",0.8 +"1986-07-01",2.8 +"1986-07-02",8.0 +"1986-07-03",9.8 +"1986-07-04",11.4 +"1986-07-05",8.6 +"1986-07-06",5.2 +"1986-07-07",6.6 +"1986-07-08",5.7 +"1986-07-09",4.6 +"1986-07-10",5.8 +"1986-07-11",7.0 +"1986-07-12",4.8 +"1986-07-13",4.4 +"1986-07-14",4.4 +"1986-07-15",7.9 +"1986-07-16",10.6 +"1986-07-17",5.0 +"1986-07-18",7.6 +"1986-07-19",9.2 +"1986-07-20",9.7 +"1986-07-21",8.8 +"1986-07-22",6.8 +"1986-07-23",9.4 +"1986-07-24",11.0 +"1986-07-25",2.5 +"1986-07-26",2.1 +"1986-07-27",5.4 +"1986-07-28",6.2 +"1986-07-29",7.8 +"1986-07-30",7.4 +"1986-07-31",9.3 +"1986-08-01",9.3 +"1986-08-02",9.5 +"1986-08-03",8.5 +"1986-08-04",10.0 +"1986-08-05",7.7 +"1986-08-06",9.3 +"1986-08-07",9.1 +"1986-08-08",3.5 +"1986-08-09",3.6 +"1986-08-10",2.5 +"1986-08-11",1.7 +"1986-08-12",2.7 +"1986-08-13",2.9 +"1986-08-14",5.3 +"1986-08-15",7.7 +"1986-08-16",9.1 +"1986-08-17",9.4 +"1986-08-18",7.3 +"1986-08-19",8.4 +"1986-08-20",9.2 +"1986-08-21",6.6 +"1986-08-22",9.7 +"1986-08-23",12.4 +"1986-08-24",10.2 +"1986-08-25",5.9 +"1986-08-26",7.1 +"1986-08-27",7.5 +"1986-08-28",9.7 +"1986-08-29",12.2 +"1986-08-30",5.6 +"1986-08-31",5.4 +"1986-09-01",8.3 +"1986-09-02",10.6 +"1986-09-03",9.1 +"1986-09-04",11.3 +"1986-09-05",10.9 +"1986-09-06",8.9 +"1986-09-07",6.3 +"1986-09-08",9.0 +"1986-09-09",6.1 +"1986-09-10",9.1 +"1986-09-11",9.6 +"1986-09-12",6.0 +"1986-09-13",10.0 +"1986-09-14",11.0 +"1986-09-15",6.2 +"1986-09-16",8.3 +"1986-09-17",11.3 +"1986-09-18",11.3 +"1986-09-19",6.7 +"1986-09-20",6.6 +"1986-09-21",11.4 +"1986-09-22",6.9 +"1986-09-23",10.6 +"1986-09-24",8.6 +"1986-09-25",11.3 +"1986-09-26",12.5 +"1986-09-27",9.9 +"1986-09-28",6.9 +"1986-09-29",5.5 +"1986-09-30",7.8 +"1986-10-01",11.0 +"1986-10-02",16.2 +"1986-10-03",9.9 +"1986-10-04",8.7 +"1986-10-05",10.5 +"1986-10-06",12.2 +"1986-10-07",10.6 +"1986-10-08",8.3 +"1986-10-09",5.5 +"1986-10-10",9.0 +"1986-10-11",6.4 +"1986-10-12",7.2 +"1986-10-13",12.9 +"1986-10-14",12.0 +"1986-10-15",7.3 +"1986-10-16",9.7 +"1986-10-17",8.4 +"1986-10-18",14.7 +"1986-10-19",9.5 +"1986-10-20",7.9 +"1986-10-21",6.8 +"1986-10-22",12.6 +"1986-10-23",5.2 +"1986-10-24",7.5 +"1986-10-25",8.7 +"1986-10-26",7.6 +"1986-10-27",9.0 +"1986-10-28",7.2 +"1986-10-29",10.7 +"1986-10-30",13.1 +"1986-10-31",13.9 +"1986-11-01",10.8 +"1986-11-02",10.4 +"1986-11-03",9.1 +"1986-11-04",16.0 +"1986-11-05",21.0 +"1986-11-06",16.2 +"1986-11-07",8.6 +"1986-11-08",9.2 +"1986-11-09",12.5 +"1986-11-10",9.7 +"1986-11-11",12.5 +"1986-11-12",10.3 +"1986-11-13",12.0 +"1986-11-14",11.0 +"1986-11-15",14.8 +"1986-11-16",15.0 +"1986-11-17",15.3 +"1986-11-18",10.3 +"1986-11-19",10.7 +"1986-11-20",10.5 +"1986-11-21",8.9 +"1986-11-22",8.1 +"1986-11-23",11.5 +"1986-11-24",12.8 +"1986-11-25",9.1 +"1986-11-26",14.6 +"1986-11-27",11.6 +"1986-11-28",11.2 +"1986-11-29",12.6 +"1986-11-30",7.5 +"1986-12-01",11.0 +"1986-12-02",14.5 +"1986-12-03",18.5 +"1986-12-04",15.4 +"1986-12-05",13.1 +"1986-12-06",16.3 +"1986-12-07",20.2 +"1986-12-08",11.5 +"1986-12-09",12.4 +"1986-12-10",10.9 +"1986-12-11",12.7 +"1986-12-12",12.2 +"1986-12-13",12.4 +"1986-12-14",9.8 +"1986-12-15",8.5 +"1986-12-16",14.7 +"1986-12-17",12.0 +"1986-12-18",10.3 +"1986-12-19",11.0 +"1986-12-20",10.2 +"1986-12-21",12.6 +"1986-12-22",11.6 +"1986-12-23",9.7 +"1986-12-24",13.4 +"1986-12-25",10.5 +"1986-12-26",14.7 +"1986-12-27",14.6 +"1986-12-28",14.2 +"1986-12-29",13.2 +"1986-12-30",11.7 +"1986-12-31",17.2 +"1987-01-01",12.3 +"1987-01-02",13.8 +"1987-01-03",15.3 +"1987-01-04",15.6 +"1987-01-05",16.2 +"1987-01-06",16.3 +"1987-01-07",16.8 +"1987-01-08",11.0 +"1987-01-09",8.5 +"1987-01-10",13.2 +"1987-01-11",13.0 +"1987-01-12",12.4 +"1987-01-13",13.0 +"1987-01-14",16.6 +"1987-01-15",12.0 +"1987-01-16",12.4 +"1987-01-17",15.0 +"1987-01-18",11.8 +"1987-01-19",11.6 +"1987-01-20",12.2 +"1987-01-21",13.7 +"1987-01-22",11.2 +"1987-01-23",12.4 +"1987-01-24",11.5 +"1987-01-25",13.8 +"1987-01-26",15.7 +"1987-01-27",12.9 +"1987-01-28",11.5 +"1987-01-29",11.0 +"1987-01-30",12.7 +"1987-01-31",14.9 +"1987-02-01",16.5 +"1987-02-02",12.8 +"1987-02-03",12.7 +"1987-02-04",12.7 +"1987-02-05",11.6 +"1987-02-06",13.3 +"1987-02-07",15.2 +"1987-02-08",16.4 +"1987-02-09",11.9 +"1987-02-10",15.1 +"1987-02-11",10.6 +"1987-02-12",13.6 +"1987-02-13",12.1 +"1987-02-14",16.0 +"1987-02-15",16.8 +"1987-02-16",16.6 +"1987-02-17",15.6 +"1987-02-18",15.2 +"1987-02-19",17.7 +"1987-02-20",21.0 +"1987-02-21",13.4 +"1987-02-22",10.5 +"1987-02-23",9.5 +"1987-02-24",12.0 +"1987-02-25",10.4 +"1987-02-26",11.5 +"1987-02-27",13.2 +"1987-02-28",15.0 +"1987-03-01",14.1 +"1987-03-02",12.4 +"1987-03-03",13.4 +"1987-03-04",12.5 +"1987-03-05",14.3 +"1987-03-06",17.6 +"1987-03-07",10.4 +"1987-03-08",9.9 +"1987-03-09",10.2 +"1987-03-10",11.3 +"1987-03-11",9.5 +"1987-03-12",11.8 +"1987-03-13",11.5 +"1987-03-14",10.5 +"1987-03-15",10.8 +"1987-03-16",13.0 +"1987-03-17",18.5 +"1987-03-18",18.7 +"1987-03-19",15.0 +"1987-03-20",13.0 +"1987-03-21",11.3 +"1987-03-22",13.0 +"1987-03-23",13.3 +"1987-03-24",11.0 +"1987-03-25",10.3 +"1987-03-26",13.0 +"1987-03-27",12.3 +"1987-03-28",15.6 +"1987-03-29",10.2 +"1987-03-30",10.8 +"1987-03-31",12.0 +"1987-04-01",13.3 +"1987-04-02",11.7 +"1987-04-03",12.5 +"1987-04-04",13.7 +"1987-04-05",14.9 +"1987-04-06",20.2 +"1987-04-07",16.3 +"1987-04-08",13.9 +"1987-04-09",10.1 +"1987-04-10",7.3 +"1987-04-11",14.0 +"1987-04-12",17.7 +"1987-04-13",16.3 +"1987-04-14",10.6 +"1987-04-15",9.7 +"1987-04-16",7.8 +"1987-04-17",10.4 +"1987-04-18",10.4 +"1987-04-19",14.1 +"1987-04-20",7.1 +"1987-04-21",8.1 +"1987-04-22",7.8 +"1987-04-23",10.6 +"1987-04-24",9.1 +"1987-04-25",9.0 +"1987-04-26",11.9 +"1987-04-27",17.1 +"1987-04-28",16.8 +"1987-04-29",13.5 +"1987-04-30",11.6 +"1987-05-01",7.0 +"1987-05-02",9.7 +"1987-05-03",9.9 +"1987-05-04",11.2 +"1987-05-05",11.3 +"1987-05-06",11.8 +"1987-05-07",9.9 +"1987-05-08",7.1 +"1987-05-09",9.6 +"1987-05-10",9.8 +"1987-05-11",10.6 +"1987-05-12",12.8 +"1987-05-13",16.5 +"1987-05-14",11.7 +"1987-05-15",12.3 +"1987-05-16",12.2 +"1987-05-17",11.8 +"1987-05-18",10.7 +"1987-05-19",10.2 +"1987-05-20",10.0 +"1987-05-21",8.3 +"1987-05-22",6.6 +"1987-05-23",9.5 +"1987-05-24",12.3 +"1987-05-25",7.6 +"1987-05-26",9.3 +"1987-05-27",5.0 +"1987-05-28",4.3 +"1987-05-29",6.4 +"1987-05-30",10.8 +"1987-05-31",7.8 +"1987-06-01",8.5 +"1987-06-02",9.7 +"1987-06-03",10.0 +"1987-06-04",11.0 +"1987-06-05",10.2 +"1987-06-06",6.6 +"1987-06-07",6.1 +"1987-06-08",5.9 +"1987-06-09",8.9 +"1987-06-10",13.0 +"1987-06-11",12.6 +"1987-06-12",5.4 +"1987-06-13",6.0 +"1987-06-14",7.8 +"1987-06-15",9.0 +"1987-06-16",4.2 +"1987-06-17",3.0 +"1987-06-18",4.5 +"1987-06-19",6.2 +"1987-06-20",11.9 +"1987-06-21",11.8 +"1987-06-22",9.4 +"1987-06-23",9.6 +"1987-06-24",9.4 +"1987-06-25",7.0 +"1987-06-26",8.9 +"1987-06-27",9.3 +"1987-06-28",6.8 +"1987-06-29",7.5 +"1987-06-30",8.0 +"1987-07-01",8.3 +"1987-07-02",2.7 +"1987-07-03",3.9 +"1987-07-04",4.1 +"1987-07-05",5.0 +"1987-07-06",5.8 +"1987-07-07",4.4 +"1987-07-08",4.1 +"1987-07-09",5.8 +"1987-07-10",9.1 +"1987-07-11",7.9 +"1987-07-12",5.0 +"1987-07-13",2.8 +"1987-07-14",4.7 +"1987-07-15",8.9 +"1987-07-16",5.4 +"1987-07-17",7.1 +"1987-07-18",9.0 +"1987-07-19",9.4 +"1987-07-20",6.3 +"1987-07-21",7.0 +"1987-07-22",6.4 +"1987-07-23",6.7 +"1987-07-24",1.5 +"1987-07-25",2.9 +"1987-07-26",4.8 +"1987-07-27",6.3 +"1987-07-28",5.7 +"1987-07-29",7.0 +"1987-07-30",8.8 +"1987-07-31",8.7 +"1987-08-01",9.0 +"1987-08-02",9.6 +"1987-08-03",8.0 +"1987-08-04",8.4 +"1987-08-05",8.1 +"1987-08-06",9.0 +"1987-08-07",5.3 +"1987-08-08",8.9 +"1987-08-09",8.7 +"1987-08-10",4.9 +"1987-08-11",7.0 +"1987-08-12",7.5 +"1987-08-13",7.0 +"1987-08-14",9.1 +"1987-08-15",11.8 +"1987-08-16",9.9 +"1987-08-17",5.6 +"1987-08-18",4.2 +"1987-08-19",4.3 +"1987-08-20",8.0 +"1987-08-21",5.1 +"1987-08-22",9.4 +"1987-08-23",9.1 +"1987-08-24",9.7 +"1987-08-25",10.6 +"1987-08-26",8.6 +"1987-08-27",10.1 +"1987-08-28",11.0 +"1987-08-29",9.7 +"1987-08-30",5.0 +"1987-08-31",6.1 +"1987-09-01",5.4 +"1987-09-02",5.8 +"1987-09-03",7.3 +"1987-09-04",6.3 +"1987-09-05",4.8 +"1987-09-06",7.6 +"1987-09-07",8.1 +"1987-09-08",9.5 +"1987-09-09",10.3 +"1987-09-10",7.0 +"1987-09-11",9.0 +"1987-09-12",10.2 +"1987-09-13",6.8 +"1987-09-14",9.3 +"1987-09-15",9.8 +"1987-09-16",10.7 +"1987-09-17",7.8 +"1987-09-18",9.2 +"1987-09-19",15.0 +"1987-09-20",7.8 +"1987-09-21",5.3 +"1987-09-22",9.5 +"1987-09-23",7.6 +"1987-09-24",14.0 +"1987-09-25",14.9 +"1987-09-26",14.9 +"1987-09-27",19.2 +"1987-09-28",17.0 +"1987-09-29",13.0 +"1987-09-30",11.2 +"1987-10-01",9.5 +"1987-10-02",10.3 +"1987-10-03",9.3 +"1987-10-04",11.3 +"1987-10-05",6.5 +"1987-10-06",12.0 +"1987-10-07",8.3 +"1987-10-08",8.7 +"1987-10-09",8.7 +"1987-10-10",10.2 +"1987-10-11",6.9 +"1987-10-12",4.9 +"1987-10-13",10.0 +"1987-10-14",7.6 +"1987-10-15",14.5 +"1987-10-16",13.2 +"1987-10-17",9.9 +"1987-10-18",10.1 +"1987-10-19",11.3 +"1987-10-20",10.4 +"1987-10-21",10.9 +"1987-10-22",9.2 +"1987-10-23",10.5 +"1987-10-24",11.4 +"1987-10-25",13.5 +"1987-10-26",9.8 +"1987-10-27",13.1 +"1987-10-28",9.7 +"1987-10-29",11.4 +"1987-10-30",9.9 +"1987-10-31",14.4 +"1987-11-01",19.0 +"1987-11-02",23.0 +"1987-11-03",15.4 +"1987-11-04",9.6 +"1987-11-05",10.8 +"1987-11-06",12.1 +"1987-11-07",11.0 +"1987-11-08",12.6 +"1987-11-09",14.7 +"1987-11-10",11.1 +"1987-11-11",10.1 +"1987-11-12",11.4 +"1987-11-13",13.0 +"1987-11-14",11.9 +"1987-11-15",9.5 +"1987-11-16",13.5 +"1987-11-17",15.2 +"1987-11-18",18.4 +"1987-11-19",24.1 +"1987-11-20",14.1 +"1987-11-21",10.7 +"1987-11-22",8.7 +"1987-11-23",13.3 +"1987-11-24",11.6 +"1987-11-25",9.9 +"1987-11-26",10.8 +"1987-11-27",11.5 +"1987-11-28",10.0 +"1987-11-29",13.9 +"1987-11-30",13.6 +"1987-12-01",11.9 +"1987-12-02",11.1 +"1987-12-03",8.2 +"1987-12-04",9.4 +"1987-12-05",12.7 +"1987-12-06",11.6 +"1987-12-07",11.0 +"1987-12-08",11.3 +"1987-12-09",13.4 +"1987-12-10",14.9 +"1987-12-11",15.2 +"1987-12-12",13.9 +"1987-12-13",15.0 +"1987-12-14",16.2 +"1987-12-15",17.7 +"1987-12-16",20.5 +"1987-12-17",14.7 +"1987-12-18",12.5 +"1987-12-19",10.9 +"1987-12-20",12.8 +"1987-12-21",12.7 +"1987-12-22",11.2 +"1987-12-23",11.4 +"1987-12-24",11.2 +"1987-12-25",12.1 +"1987-12-26",12.7 +"1987-12-27",16.2 +"1987-12-28",14.2 +"1987-12-29",14.3 +"1987-12-30",13.3 +"1987-12-31",16.7 +"1988-01-01",15.3 +"1988-01-02",14.3 +"1988-01-03",13.5 +"1988-01-04",15.0 +"1988-01-05",13.6 +"1988-01-06",15.2 +"1988-01-07",17.0 +"1988-01-08",18.7 +"1988-01-09",16.5 +"1988-01-10",17.4 +"1988-01-11",18.3 +"1988-01-12",18.3 +"1988-01-13",22.4 +"1988-01-14",21.4 +"1988-01-15",20.9 +"1988-01-16",17.6 +"1988-01-17",15.5 +"1988-01-18",16.6 +"1988-01-19",16.2 +"1988-01-20",15.6 +"1988-01-21",14.5 +"1988-01-22",14.0 +"1988-01-23",15.6 +"1988-01-24",12.3 +"1988-01-25",11.6 +"1988-01-26",12.6 +"1988-01-27",14.9 +"1988-01-28",17.3 +"1988-01-29",21.4 +"1988-01-30",23.4 +"1988-01-31",14.4 +"1988-02-01",14.1 +"1988-02-02",15.0 +"1988-02-03",14.5 +"1988-02-04",15.1 +"1988-02-05",13.9 +"1988-02-06",13.4 +"1988-02-07",9.2 +"1988-02-08",12.5 +"1988-02-09",15.1 +"1988-02-10",12.1 +"1988-02-11",14.5 +"1988-02-12",16.3 +"1988-02-13",16.5 +"1988-02-14",14.9 +"1988-02-15",13.2 +"1988-02-16",11.8 +"1988-02-17",13.6 +"1988-02-18",16.2 +"1988-02-19",14.1 +"1988-02-20",13.5 +"1988-02-21",15.0 +"1988-02-22",14.8 +"1988-02-23",16.2 +"1988-02-24",16.2 +"1988-02-25",13.3 +"1988-02-26",15.3 +"1988-02-27",18.4 +"1988-02-28",16.2 +"1988-02-29",16.3 +"1988-03-01",12.4 +"1988-03-02",15.6 +"1988-03-03",14.9 +"1988-03-04",14.8 +"1988-03-05",12.7 +"1988-03-06",14.2 +"1988-03-07",16.8 +"1988-03-08",16.7 +"1988-03-09",16.2 +"1988-03-10",14.5 +"1988-03-11",10.0 +"1988-03-12",12.6 +"1988-03-13",11.9 +"1988-03-14",11.8 +"1988-03-15",13.4 +"1988-03-16",14.5 +"1988-03-17",15.7 +"1988-03-18",15.3 +"1988-03-19",13.9 +"1988-03-20",13.7 +"1988-03-21",15.1 +"1988-03-22",15.6 +"1988-03-23",14.4 +"1988-03-24",13.9 +"1988-03-25",16.2 +"1988-03-26",16.7 +"1988-03-27",15.5 +"1988-03-28",16.4 +"1988-03-29",17.5 +"1988-03-30",18.2 +"1988-03-31",16.1 +"1988-04-01",16.5 +"1988-04-02",14.6 +"1988-04-03",16.4 +"1988-04-04",13.6 +"1988-04-05",15.9 +"1988-04-06",11.9 +"1988-04-07",14.7 +"1988-04-08",9.4 +"1988-04-09",6.6 +"1988-04-10",7.9 +"1988-04-11",11.0 +"1988-04-12",15.7 +"1988-04-13",15.2 +"1988-04-14",15.9 +"1988-04-15",10.6 +"1988-04-16",8.3 +"1988-04-17",8.6 +"1988-04-18",12.7 +"1988-04-19",10.5 +"1988-04-20",12.0 +"1988-04-21",11.1 +"1988-04-22",13.0 +"1988-04-23",12.4 +"1988-04-24",13.3 +"1988-04-25",15.9 +"1988-04-26",12.0 +"1988-04-27",13.7 +"1988-04-28",17.6 +"1988-04-29",14.3 +"1988-04-30",13.7 +"1988-05-01",15.2 +"1988-05-02",14.5 +"1988-05-03",14.9 +"1988-05-04",15.5 +"1988-05-05",16.4 +"1988-05-06",14.5 +"1988-05-07",12.6 +"1988-05-08",13.6 +"1988-05-09",11.2 +"1988-05-10",11.0 +"1988-05-11",12.0 +"1988-05-12",6.8 +"1988-05-13",10.6 +"1988-05-14",13.1 +"1988-05-15",13.5 +"1988-05-16",11.7 +"1988-05-17",13.2 +"1988-05-18",12.0 +"1988-05-19",10.4 +"1988-05-20",10.0 +"1988-05-21",8.2 +"1988-05-22",9.4 +"1988-05-23",10.3 +"1988-05-24",8.1 +"1988-05-25",8.7 +"1988-05-26",12.6 +"1988-05-27",10.9 +"1988-05-28",8.7 +"1988-05-29",9.3 +"1988-05-30",6.3 +"1988-05-31",7.8 +"1988-06-01",10.0 +"1988-06-02",11.0 +"1988-06-03",11.1 +"1988-06-04",12.6 +"1988-06-05",10.2 +"1988-06-06",11.1 +"1988-06-07",8.7 +"1988-06-08",9.5 +"1988-06-09",9.7 +"1988-06-10",8.2 +"1988-06-11",5.0 +"1988-06-12",6.5 +"1988-06-13",12.1 +"1988-06-14",8.9 +"1988-06-15",6.1 +"1988-06-16",2.8 +"1988-06-17",3.7 +"1988-06-18",6.8 +"1988-06-19",6.6 +"1988-06-20",7.0 +"1988-06-21",7.3 +"1988-06-22",7.9 +"1988-06-23",10.6 +"1988-06-24",8.1 +"1988-06-25",6.7 +"1988-06-26",8.0 +"1988-06-27",10.0 +"1988-06-28",6.7 +"1988-06-29",9.4 +"1988-06-30",9.3 +"1988-07-01",6.0 +"1988-07-02",5.8 +"1988-07-03",4.9 +"1988-07-04",5.0 +"1988-07-05",8.4 +"1988-07-06",12.3 +"1988-07-07",13.0 +"1988-07-08",11.4 +"1988-07-09",6.8 +"1988-07-10",7.6 +"1988-07-11",12.4 +"1988-07-12",7.1 +"1988-07-13",7.5 +"1988-07-14",10.0 +"1988-07-15",5.3 +"1988-07-16",6.3 +"1988-07-17",8.0 +"1988-07-18",8.3 +"1988-07-19",9.3 +"1988-07-20",9.5 +"1988-07-21",5.6 +"1988-07-22",7.0 +"1988-07-23",8.5 +"1988-07-24",8.5 +"1988-07-25",8.2 +"1988-07-26",8.5 +"1988-07-27",9.6 +"1988-07-28",9.7 +"1988-07-29",7.1 +"1988-07-30",8.4 +"1988-07-31",9.2 +"1988-08-01",9.8 +"1988-08-02",8.1 +"1988-08-03",9.4 +"1988-08-04",10.0 +"1988-08-05",5.1 +"1988-08-06",6.7 +"1988-08-07",6.9 +"1988-08-08",6.8 +"1988-08-09",8.6 +"1988-08-10",9.1 +"1988-08-11",3.9 +"1988-08-12",4.8 +"1988-08-13",8.4 +"1988-08-14",11.6 +"1988-08-15",12.1 +"1988-08-16",12.4 +"1988-08-17",10.0 +"1988-08-18",10.1 +"1988-08-19",9.7 +"1988-08-20",11.7 +"1988-08-21",7.9 +"1988-08-22",8.6 +"1988-08-23",7.7 +"1988-08-24",5.8 +"1988-08-25",8.7 +"1988-08-26",10.6 +"1988-08-27",6.7 +"1988-08-28",8.8 +"1988-08-29",9.7 +"1988-08-30",9.0 +"1988-08-31",11.8 +"1988-09-01",15.2 +"1988-09-02",10.0 +"1988-09-03",10.5 +"1988-09-04",5.5 +"1988-09-05",9.4 +"1988-09-06",8.8 +"1988-09-07",5.3 +"1988-09-08",13.0 +"1988-09-09",15.2 +"1988-09-10",13.2 +"1988-09-11",11.5 +"1988-09-12",6.8 +"1988-09-13",4.7 +"1988-09-14",5.2 +"1988-09-15",6.8 +"1988-09-16",10.7 +"1988-09-17",10.1 +"1988-09-18",10.0 +"1988-09-19",9.8 +"1988-09-20",5.5 +"1988-09-21",13.5 +"1988-09-22",16.6 +"1988-09-23",8.4 +"1988-09-24",8.2 +"1988-09-25",11.1 +"1988-09-26",10.8 +"1988-09-27",8.8 +"1988-09-28",10.8 +"1988-09-29",8.7 +"1988-09-30",12.4 +"1988-10-01",9.0 +"1988-10-02",13.5 +"1988-10-03",14.7 +"1988-10-04",10.9 +"1988-10-05",8.5 +"1988-10-06",6.0 +"1988-10-07",12.7 +"1988-10-08",11.1 +"1988-10-09",8.7 +"1988-10-10",12.3 +"1988-10-11",13.3 +"1988-10-12",5.6 +"1988-10-13",13.7 +"1988-10-14",8.5 +"1988-10-15",11.2 +"1988-10-16",8.7 +"1988-10-17",11.7 +"1988-10-18",12.5 +"1988-10-19",8.2 +"1988-10-20",15.6 +"1988-10-21",10.3 +"1988-10-22",11.4 +"1988-10-23",9.7 +"1988-10-24",6.3 +"1988-10-25",14.3 +"1988-10-26",11.3 +"1988-10-27",7.3 +"1988-10-28",12.8 +"1988-10-29",11.9 +"1988-10-30",14.3 +"1988-10-31",11.6 +"1988-11-01",13.2 +"1988-11-02",15.5 +"1988-11-03",14.1 +"1988-11-04",9.5 +"1988-11-05",7.2 +"1988-11-06",11.8 +"1988-11-07",16.8 +"1988-11-08",12.5 +"1988-11-09",9.4 +"1988-11-10",11.9 +"1988-11-11",10.3 +"1988-11-12",16.9 +"1988-11-13",17.5 +"1988-11-14",7.5 +"1988-11-15",8.6 +"1988-11-16",11.1 +"1988-11-17",11.5 +"1988-11-18",10.7 +"1988-11-19",15.7 +"1988-11-20",12.8 +"1988-11-21",13.0 +"1988-11-22",12.9 +"1988-11-23",14.3 +"1988-11-24",13.7 +"1988-11-25",12.1 +"1988-11-26",11.9 +"1988-11-27",11.8 +"1988-11-28",11.4 +"1988-11-29",10.3 +"1988-11-30",11.7 +"1988-12-01",12.0 +"1988-12-02",17.4 +"1988-12-03",16.8 +"1988-12-04",16.2 +"1988-12-05",13.0 +"1988-12-06",12.5 +"1988-12-07",12.4 +"1988-12-08",16.1 +"1988-12-09",20.2 +"1988-12-10",14.3 +"1988-12-11",11.0 +"1988-12-12",14.4 +"1988-12-13",15.7 +"1988-12-14",19.7 +"1988-12-15",20.7 +"1988-12-16",23.9 +"1988-12-17",16.6 +"1988-12-18",17.5 +"1988-12-19",14.9 +"1988-12-20",13.6 +"1988-12-21",11.9 +"1988-12-22",15.2 +"1988-12-23",17.3 +"1988-12-24",19.8 +"1988-12-25",15.8 +"1988-12-26",9.5 +"1988-12-27",12.9 +"1988-12-28",12.9 +"1988-12-29",14.8 +"1988-12-30",14.1 +"1989-01-01",14.3 +"1989-01-02",17.4 +"1989-01-03",18.5 +"1989-01-04",16.8 +"1989-01-05",11.5 +"1989-01-06",9.5 +"1989-01-07",12.2 +"1989-01-08",15.7 +"1989-01-09",16.3 +"1989-01-10",13.6 +"1989-01-11",12.6 +"1989-01-12",13.8 +"1989-01-13",12.1 +"1989-01-14",13.4 +"1989-01-15",17.3 +"1989-01-16",19.4 +"1989-01-17",16.6 +"1989-01-18",13.9 +"1989-01-19",13.1 +"1989-01-20",16.0 +"1989-01-21",14.5 +"1989-01-22",15.0 +"1989-01-23",12.6 +"1989-01-24",12.5 +"1989-01-25",15.2 +"1989-01-26",16.2 +"1989-01-27",16.5 +"1989-01-28",20.1 +"1989-01-29",20.6 +"1989-01-30",16.9 +"1989-01-31",16.5 +"1989-02-01",16.1 +"1989-02-02",14.4 +"1989-02-03",16.3 +"1989-02-04",15.7 +"1989-02-05",14.2 +"1989-02-06",13.2 +"1989-02-07",16.8 +"1989-02-08",18.5 +"1989-02-09",16.7 +"1989-02-10",15.3 +"1989-02-11",15.9 +"1989-02-12",15.2 +"1989-02-13",17.5 +"1989-02-14",18.3 +"1989-02-15",19.4 +"1989-02-16",19.4 +"1989-02-17",19.5 +"1989-02-18",20.5 +"1989-02-19",15.7 +"1989-02-20",15.0 +"1989-02-21",16.1 +"1989-02-22",14.3 +"1989-02-23",13.0 +"1989-02-24",16.2 +"1989-02-25",17.7 +"1989-02-26",13.2 +"1989-02-27",15.8 +"1989-02-28",18.5 +"1989-03-01",20.4 +"1989-03-02",22.0 +"1989-03-03",19.7 +"1989-03-04",19.6 +"1989-03-05",20.3 +"1989-03-06",18.3 +"1989-03-07",18.9 +"1989-03-08",20.3 +"1989-03-09",21.4 +"1989-03-10",18.3 +"1989-03-11",17.8 +"1989-03-12",17.7 +"1989-03-13",12.8 +"1989-03-14",15.1 +"1989-03-15",15.0 +"1989-03-16",14.8 +"1989-03-17",12.0 +"1989-03-18",12.5 +"1989-03-19",15.0 +"1989-03-20",17.1 +"1989-03-21",17.3 +"1989-03-22",16.9 +"1989-03-23",16.5 +"1989-03-24",13.6 +"1989-03-25",13.2 +"1989-03-26",9.4 +"1989-03-27",9.5 +"1989-03-28",11.8 +"1989-03-29",10.4 +"1989-03-30",9.7 +"1989-03-31",12.6 +"1989-04-01",13.3 +"1989-04-02",15.1 +"1989-04-03",14.2 +"1989-04-04",14.2 +"1989-04-05",19.2 +"1989-04-06",12.6 +"1989-04-07",14.2 +"1989-04-08",11.9 +"1989-04-09",13.9 +"1989-04-10",13.5 +"1989-04-11",15.3 +"1989-04-12",13.9 +"1989-04-13",14.0 +"1989-04-14",12.9 +"1989-04-15",8.5 +"1989-04-16",11.4 +"1989-04-17",10.9 +"1989-04-18",12.0 +"1989-04-19",8.6 +"1989-04-20",9.0 +"1989-04-21",9.6 +"1989-04-22",10.2 +"1989-04-23",9.8 +"1989-04-24",8.3 +"1989-04-25",11.0 +"1989-04-26",11.9 +"1989-04-27",14.0 +"1989-04-28",15.8 +"1989-04-29",14.5 +"1989-04-30",13.2 +"1989-05-01",14.2 +"1989-05-02",14.6 +"1989-05-03",11.8 +"1989-05-04",14.4 +"1989-05-05",10.4 +"1989-05-06",10.3 +"1989-05-07",10.8 +"1989-05-08",10.5 +"1989-05-09",9.5 +"1989-05-10",12.5 +"1989-05-11",13.7 +"1989-05-12",12.7 +"1989-05-13",11.9 +"1989-05-14",11.4 +"1989-05-15",9.7 +"1989-05-16",8.3 +"1989-05-17",8.1 +"1989-05-18",11.7 +"1989-05-19",11.6 +"1989-05-20",7.4 +"1989-05-21",5.2 +"1989-05-22",11.0 +"1989-05-23",9.5 +"1989-05-24",9.2 +"1989-05-25",10.7 +"1989-05-26",9.0 +"1989-05-27",10.2 +"1989-05-28",10.3 +"1989-05-29",12.1 +"1989-05-30",13.2 +"1989-05-31",6.6 +"1989-06-01",2.3 +"1989-06-02",1.4 +"1989-06-03",2.1 +"1989-06-04",6.6 +"1989-06-05",8.9 +"1989-06-06",7.8 +"1989-06-07",9.0 +"1989-06-08",10.3 +"1989-06-09",7.9 +"1989-06-10",7.2 +"1989-06-11",8.6 +"1989-06-12",8.8 +"1989-06-13",6.2 +"1989-06-14",9.5 +"1989-06-15",10.2 +"1989-06-16",9.7 +"1989-06-17",11.2 +"1989-06-18",10.2 +"1989-06-19",10.1 +"1989-06-20",8.1 +"1989-06-21",6.6 +"1989-06-22",5.0 +"1989-06-23",4.7 +"1989-06-24",5.3 +"1989-06-25",4.5 +"1989-06-26",2.3 +"1989-06-27",1.4 +"1989-06-28",0.5 +"1989-06-29",2.4 +"1989-06-30",8.0 +"1989-07-01",6.0 +"1989-07-02",7.1 +"1989-07-03",9.7 +"1989-07-04",6.9 +"1989-07-05",5.3 +"1989-07-06",7.0 +"1989-07-07",6.2 +"1989-07-08",7.0 +"1989-07-09",9.7 +"1989-07-10",8.0 +"1989-07-11",8.5 +"1989-07-12",7.1 +"1989-07-13",7.5 +"1989-07-14",3.3 +"1989-07-15",1.8 +"1989-07-16",2.6 +"1989-07-17",5.3 +"1989-07-18",5.8 +"1989-07-19",5.8 +"1989-07-20",7.2 +"1989-07-21",5.3 +"1989-07-22",1.6 +"1989-07-23",3.1 +"1989-07-24",5.3 +"1989-07-25",7.7 +"1989-07-26",4.2 +"1989-07-27",5.5 +"1989-07-28",9.0 +"1989-07-29",11.2 +"1989-07-30",8.0 +"1989-07-31",7.6 +"1989-08-01",3.7 +"1989-08-02",7.5 +"1989-08-03",8.1 +"1989-08-04",8.4 +"1989-08-05",7.1 +"1989-08-06",7.6 +"1989-08-07",7.6 +"1989-08-08",5.6 +"1989-08-09",7.0 +"1989-08-10",10.5 +"1989-08-11",7.3 +"1989-08-12",7.8 +"1989-08-13",5.8 +"1989-08-14",3.8 +"1989-08-15",5.8 +"1989-08-16",6.7 +"1989-08-17",6.6 +"1989-08-18",6.6 +"1989-08-19",9.0 +"1989-08-20",8.1 +"1989-08-21",5.1 +"1989-08-22",8.6 +"1989-08-23",7.0 +"1989-08-24",5.5 +"1989-08-25",7.4 +"1989-08-26",6.2 +"1989-08-27",4.2 +"1989-08-28",6.3 +"1989-08-29",7.0 +"1989-08-30",4.0 +"1989-08-31",8.0 +"1989-09-01",8.8 +"1989-09-02",8.8 +"1989-09-03",6.1 +"1989-09-04",8.6 +"1989-09-05",8.9 +"1989-09-06",7.8 +"1989-09-07",5.0 +"1989-09-08",7.0 +"1989-09-09",13.3 +"1989-09-10",7.9 +"1989-09-11",7.5 +"1989-09-12",8.3 +"1989-09-13",7.2 +"1989-09-14",6.5 +"1989-09-15",8.9 +"1989-09-16",7.4 +"1989-09-17",9.9 +"1989-09-18",9.3 +"1989-09-19",10.6 +"1989-09-20",8.6 +"1989-09-21",7.2 +"1989-09-22",12.6 +"1989-09-23",7.8 +"1989-09-24",6.3 +"1989-09-25",9.2 +"1989-09-26",5.8 +"1989-09-27",9.0 +"1989-09-28",5.0 +"1989-09-29",11.9 +"1989-09-30",13.4 +"1989-10-01",10.5 +"1989-10-02",6.2 +"1989-10-03",5.1 +"1989-10-04",9.5 +"1989-10-05",11.7 +"1989-10-06",9.2 +"1989-10-07",7.3 +"1989-10-08",9.7 +"1989-10-09",9.4 +"1989-10-10",10.0 +"1989-10-11",10.9 +"1989-10-12",11.0 +"1989-10-13",10.9 +"1989-10-14",8.0 +"1989-10-15",11.2 +"1989-10-16",7.5 +"1989-10-17",7.2 +"1989-10-18",13.2 +"1989-10-19",12.9 +"1989-10-20",9.4 +"1989-10-21",10.2 +"1989-10-22",9.5 +"1989-10-23",12.4 +"1989-10-24",10.2 +"1989-10-25",13.4 +"1989-10-26",11.6 +"1989-10-27",8.0 +"1989-10-28",9.0 +"1989-10-29",9.3 +"1989-10-30",13.5 +"1989-10-31",8.0 +"1989-11-01",8.1 +"1989-11-02",10.0 +"1989-11-03",8.5 +"1989-11-04",12.5 +"1989-11-05",15.0 +"1989-11-06",13.3 +"1989-11-07",11.0 +"1989-11-08",11.9 +"1989-11-09",8.3 +"1989-11-10",9.7 +"1989-11-11",11.3 +"1989-11-12",12.5 +"1989-11-13",9.4 +"1989-11-14",11.4 +"1989-11-15",13.2 +"1989-11-16",13.8 +"1989-11-17",16.0 +"1989-11-18",10.9 +"1989-11-19",11.9 +"1989-11-20",12.4 +"1989-11-21",13.2 +"1989-11-22",15.5 +"1989-11-23",21.6 +"1989-11-24",14.9 +"1989-11-25",14.4 +"1989-11-26",12.9 +"1989-11-27",13.1 +"1989-11-28",14.0 +"1989-11-29",17.9 +"1989-11-30",17.7 +"1989-12-01",16.3 +"1989-12-02",18.3 +"1989-12-03",13.7 +"1989-12-04",13.3 +"1989-12-05",10.6 +"1989-12-06",14.1 +"1989-12-07",16.0 +"1989-12-08",16.5 +"1989-12-09",14.1 +"1989-12-10",18.7 +"1989-12-11",16.2 +"1989-12-12",14.8 +"1989-12-13",12.6 +"1989-12-14",10.4 +"1989-12-15",12.2 +"1989-12-16",12.6 +"1989-12-17",12.1 +"1989-12-18",17.3 +"1989-12-19",16.4 +"1989-12-20",12.6 +"1989-12-21",12.3 +"1989-12-22",11.8 +"1989-12-23",12.0 +"1989-12-24",12.7 +"1989-12-25",16.4 +"1989-12-26",16.0 +"1989-12-27",13.3 +"1989-12-28",11.7 +"1989-12-29",10.4 +"1989-12-30",14.4 +"1989-12-31",12.7 +"1990-01-01",14.8 +"1990-01-02",13.3 +"1990-01-03",15.6 +"1990-01-04",14.5 +"1990-01-05",14.3 +"1990-01-06",15.3 +"1990-01-07",16.4 +"1990-01-08",14.8 +"1990-01-09",17.4 +"1990-01-10",18.8 +"1990-01-11",22.1 +"1990-01-12",19.0 +"1990-01-13",15.5 +"1990-01-14",15.8 +"1990-01-15",14.7 +"1990-01-16",10.7 +"1990-01-17",11.5 +"1990-01-18",15.0 +"1990-01-19",14.5 +"1990-01-20",14.5 +"1990-01-21",13.3 +"1990-01-22",14.3 +"1990-01-23",14.3 +"1990-01-24",20.5 +"1990-01-25",15.0 +"1990-01-26",17.1 +"1990-01-27",16.9 +"1990-01-28",16.9 +"1990-01-29",13.6 +"1990-01-30",16.4 +"1990-01-31",16.1 +"1990-02-01",12.0 +"1990-02-02",12.2 +"1990-02-03",14.8 +"1990-02-04",14.8 +"1990-02-05",14.4 +"1990-02-06",12.9 +"1990-02-07",13.4 +"1990-02-08",15.9 +"1990-02-09",16.1 +"1990-02-10",17.6 +"1990-02-11",15.6 +"1990-02-12",15.0 +"1990-02-13",13.0 +"1990-02-14",14.1 +"1990-02-15",17.3 +"1990-02-16",15.7 +"1990-02-17",18.6 +"1990-02-18",12.7 +"1990-02-19",14.0 +"1990-02-20",13.7 +"1990-02-21",16.3 +"1990-02-22",20.0 +"1990-02-23",17.0 +"1990-02-24",15.2 +"1990-02-25",16.5 +"1990-02-26",16.5 +"1990-02-27",17.3 +"1990-02-28",19.1 +"1990-03-01",19.3 +"1990-03-02",17.3 +"1990-03-03",19.0 +"1990-03-04",19.8 +"1990-03-05",19.3 +"1990-03-06",17.2 +"1990-03-07",14.2 +"1990-03-08",10.3 +"1990-03-09",13.0 +"1990-03-10",15.3 +"1990-03-11",15.0 +"1990-03-12",12.1 +"1990-03-13",9.2 +"1990-03-14",11.0 +"1990-03-15",15.0 +"1990-03-16",11.6 +"1990-03-17",11.6 +"1990-03-18",15.1 +"1990-03-19",15.0 +"1990-03-20",13.6 +"1990-03-21",12.5 +"1990-03-22",14.3 +"1990-03-23",16.0 +"1990-03-24",17.4 +"1990-03-25",16.9 +"1990-03-26",18.0 +"1990-03-27",20.6 +"1990-03-28",14.2 +"1990-03-29",10.9 +"1990-03-30",11.9 +"1990-03-31",13.3 +"1990-04-01",15.3 +"1990-04-02",14.7 +"1990-04-03",11.0 +"1990-04-04",12.2 +"1990-04-05",14.2 +"1990-04-06",17.0 +"1990-04-07",15.8 +"1990-04-08",15.2 +"1990-04-09",15.1 +"1990-04-10",14.7 +"1990-04-11",18.5 +"1990-04-12",16.4 +"1990-04-13",18.4 +"1990-04-14",15.1 +"1990-04-15",9.9 +"1990-04-16",10.2 +"1990-04-17",12.6 +"1990-04-18",13.2 +"1990-04-19",11.5 +"1990-04-20",13.8 +"1990-04-21",14.5 +"1990-04-22",14.7 +"1990-04-23",11.2 +"1990-04-24",12.7 +"1990-04-25",13.7 +"1990-04-26",11.5 +"1990-04-27",10.4 +"1990-04-28",8.9 +"1990-04-29",11.1 +"1990-04-30",9.5 +"1990-05-01",13.0 +"1990-05-02",13.9 +"1990-05-03",12.6 +"1990-05-04",14.3 +"1990-05-05",16.0 +"1990-05-06",13.3 +"1990-05-07",7.0 +"1990-05-08",4.9 +"1990-05-09",6.9 +"1990-05-10",13.7 +"1990-05-11",10.6 +"1990-05-12",12.3 +"1990-05-13",11.1 +"1990-05-14",10.2 +"1990-05-15",9.5 +"1990-05-16",8.9 +"1990-05-17",13.4 +"1990-05-18",9.1 +"1990-05-19",9.4 +"1990-05-20",8.7 +"1990-05-21",5.8 +"1990-05-22",4.5 +"1990-05-23",7.2 +"1990-05-24",10.0 +"1990-05-25",10.5 +"1990-05-26",10.7 +"1990-05-27",8.2 +"1990-05-28",6.1 +"1990-05-29",4.5 +"1990-05-30",6.1 +"1990-05-31",9.8 +"1990-06-01",9.7 +"1990-06-02",8.2 +"1990-06-03",8.4 +"1990-06-04",8.5 +"1990-06-05",10.4 +"1990-06-06",6.8 +"1990-06-07",6.0 +"1990-06-08",6.6 +"1990-06-09",7.8 +"1990-06-10",10.3 +"1990-06-11",7.2 +"1990-06-12",7.4 +"1990-06-13",11.4 +"1990-06-14",5.4 +"1990-06-15",4.4 +"1990-06-16",6.4 +"1990-06-17",9.3 +"1990-06-18",7.7 +"1990-06-19",8.1 +"1990-06-20",8.3 +"1990-06-21",9.1 +"1990-06-22",7.7 +"1990-06-23",10.6 +"1990-06-24",8.2 +"1990-06-25",7.9 +"1990-06-26",5.2 +"1990-06-27",5.9 +"1990-06-28",3.7 +"1990-06-29",5.6 +"1990-06-30",9.4 +"1990-07-01",7.4 +"1990-07-02",7.3 +"1990-07-03",7.7 +"1990-07-04",7.7 +"1990-07-05",9.3 +"1990-07-06",4.4 +"1990-07-07",5.7 +"1990-07-08",10.2 +"1990-07-09",10.2 +"1990-07-10",9.3 +"1990-07-11",5.4 +"1990-07-12",5.0 +"1990-07-13",7.6 +"1990-07-14",9.6 +"1990-07-15",10.4 +"1990-07-16",11.2 +"1990-07-17",9.1 +"1990-07-18",11.2 +"1990-07-19",6.8 +"1990-07-20",8.3 +"1990-07-21",9.7 +"1990-07-22",9.6 +"1990-07-23",9.8 +"1990-07-24",10.8 +"1990-07-25",9.2 +"1990-07-26",6.5 +"1990-07-27",8.1 +"1990-07-28",7.3 +"1990-07-29",7.9 +"1990-07-30",6.0 +"1990-07-31",5.0 +"1990-08-01",6.8 +"1990-08-02",9.8 +"1990-08-03",5.7 +"1990-08-04",8.6 +"1990-08-05",10.6 +"1990-08-06",7.8 +"1990-08-07",7.7 +"1990-08-08",8.6 +"1990-08-09",6.5 +"1990-08-10",6.9 +"1990-08-11",6.4 +"1990-08-12",8.5 +"1990-08-13",7.8 +"1990-08-14",9.3 +"1990-08-15",8.4 +"1990-08-16",7.8 +"1990-08-17",7.4 +"1990-08-18",7.7 +"1990-08-19",8.9 +"1990-08-20",9.7 +"1990-08-21",9.9 +"1990-08-22",6.1 +"1990-08-23",6.6 +"1990-08-24",7.6 +"1990-08-25",7.4 +"1990-08-26",8.0 +"1990-08-27",2.1 +"1990-08-28",5.9 +"1990-08-29",11.6 +"1990-08-30",8.6 +"1990-08-31",7.9 +"1990-09-01",6.0 +"1990-09-02",9.5 +"1990-09-03",8.6 +"1990-09-04",7.6 +"1990-09-05",10.4 +"1990-09-06",10.3 +"1990-09-07",7.5 +"1990-09-08",3.0 +"1990-09-09",5.3 +"1990-09-10",10.5 +"1990-09-11",14.6 +"1990-09-12",12.6 +"1990-09-13",9.8 +"1990-09-14",7.2 +"1990-09-15",10.1 +"1990-09-16",10.4 +"1990-09-17",3.7 +"1990-09-18",7.3 +"1990-09-19",11.6 +"1990-09-20",16.3 +"1990-09-21",9.6 +"1990-09-22",6.8 +"1990-09-23",5.2 +"1990-09-24",10.6 +"1990-09-25",16.3 +"1990-09-26",9.8 +"1990-09-27",4.6 +"1990-09-28",11.1 +"1990-09-29",8.7 +"1990-09-30",10.0 +"1990-10-01",11.3 +"1990-10-02",10.5 +"1990-10-03",9.9 +"1990-10-04",11.0 +"1990-10-05",14.0 +"1990-10-06",9.2 +"1990-10-07",9.8 +"1990-10-08",6.0 +"1990-10-09",9.8 +"1990-10-10",9.2 +"1990-10-11",11.8 +"1990-10-12",10.3 +"1990-10-13",7.5 +"1990-10-14",7.7 +"1990-10-15",15.8 +"1990-10-16",14.6 +"1990-10-17",10.5 +"1990-10-18",11.3 +"1990-10-19",10.9 +"1990-10-20",6.4 +"1990-10-21",10.9 +"1990-10-22",9.0 +"1990-10-23",10.9 +"1990-10-24",12.4 +"1990-10-25",11.6 +"1990-10-26",13.3 +"1990-10-27",14.4 +"1990-10-28",18.4 +"1990-10-29",13.6 +"1990-10-30",14.9 +"1990-10-31",14.8 +"1990-11-01",15.4 +"1990-11-02",11.8 +"1990-11-03",13.0 +"1990-11-04",11.1 +"1990-11-05",12.5 +"1990-11-06",18.3 +"1990-11-07",19.2 +"1990-11-08",15.4 +"1990-11-09",13.1 +"1990-11-10",11.5 +"1990-11-11",8.6 +"1990-11-12",12.6 +"1990-11-13",13.8 +"1990-11-14",14.6 +"1990-11-15",13.2 +"1990-11-16",12.3 +"1990-11-17",8.8 +"1990-11-18",10.7 +"1990-11-19",9.9 +"1990-11-20",8.3 +"1990-11-21",15.0 +"1990-11-22",12.2 +"1990-11-23",10.5 +"1990-11-24",11.1 +"1990-11-25",13.0 +"1990-11-26",12.9 +"1990-11-27",8.8 +"1990-11-28",14.7 +"1990-11-29",14.7 +"1990-11-30",12.7 +"1990-12-01",13.3 +"1990-12-02",13.2 +"1990-12-03",16.2 +"1990-12-04",17.3 +"1990-12-05",20.5 +"1990-12-06",20.2 +"1990-12-07",19.4 +"1990-12-08",15.5 +"1990-12-09",14.1 +"1990-12-10",11.0 +"1990-12-11",11.1 +"1990-12-12",14.0 +"1990-12-13",11.4 +"1990-12-14",12.5 +"1990-12-15",13.4 +"1990-12-16",13.6 +"1990-12-17",13.9 +"1990-12-18",17.2 +"1990-12-19",14.7 +"1990-12-20",15.4 +"1990-12-21",13.1 +"1990-12-22",13.2 +"1990-12-23",13.9 +"1990-12-24",10.0 +"1990-12-25",12.9 +"1990-12-26",14.6 +"1990-12-27",14.0 +"1990-12-28",13.6 +"1990-12-29",13.5 +"1990-12-30",15.7 +"1990-12-31",13.0 diff --git a/docs/adr/0002-explicit-identification-and-claim-blockers.md b/docs/adr/0002-explicit-identification-and-claim-blockers.md new file mode 100644 index 0000000..fdb8dfa --- /dev/null +++ b/docs/adr/0002-explicit-identification-and-claim-blockers.md @@ -0,0 +1,71 @@ +# ADR 0002 — Explicit identification assumptions and claim-scoped blockers + +Status: implemented; CI review pending. + +## Context + +The assessment runner treated observed covariate overlap as exchangeability, +accepted overlap in just one stratum, and could allow claims despite integrity +blockers. Its graph attached unrelated findings to the positivity check. These +contradicted INV-001, INV-002, INV-008 and INV-010. + +## Decision + +- Empirical support requires exactly two non-null treatment levels in every + declared target stratum, with non-null covariates. This conservative discrete + check does not estimate population positivity. Continuous covariates need an + explicitly reviewed upstream coarsening; automatic binning is not performed. +- `CausalContract.identification_assumptions` is an optional tuple of named + assumptions. Existing contracts deserialize with an empty tuple. Free-form + prose and overlap do not automatically acknowledge an identifying assumption. +- Missing identifying conditions are repairable missing evidence, not proof of + fundamental non-identifiability. Empirical lack of support remains a design + blocker. A declared assumption cannot override that blocker or a mismatch + between the adjustment set and checked columns. +- Conditional identification produces `ANSWERABLE_WITH_ASSUMPTIONS`. No table + check verifies random assignment, absence of unmeasured confounding, parallel + trends, instrument validity or discontinuity validity merely because a user + selected a strategy. The observed mean difference is still descriptive; this + change does not implement adjusted causal estimation. +- Facts, declared assumptions and conditions not verifiable from the table are + separate graph nodes and separate lists in + `data_quality_relevance.evidence_status` in the warrant. Existing canonical + top-level warrant sections remain unchanged. Markdown displays all three. +- Integrity/execution blockers affect all claims; other inference blockers + prevent causal/predictive/diagnostic/recommendation claims regardless of + wording. Raw descriptive associations may survive missing causal evidence. + `causal_gate` is checked by claim class, not only an English verb regex. +- Each finding links to its actual producing check; observations link to the + means computation. Plans/manifests and graph execution nodes use the same + selected registry. Only the primary dataset is linked to these checks because + this runner currently computes on the first dataset. +- Pure descriptive questions do not require causal identification or inferential + power. Adding a causal candidate still activates causal checks. +- Assessment identity includes the complete mapping, causal contract and claim + candidates. Changing acknowledged assumptions therefore changes identity. + +## Compatibility and evidence + +The v1 causal JSON schema accepts the new optional field; legacy payloads remain +valid. Graph schemas already accept string node types. Old warrant hashes are +not rewritten. Existing tests that equated overlap with unconditional support +now explicitly declare assumptions and assert the stricter conditional verdict. +No coverage or quality threshold is lowered. + +EMT-v1/v2 frozen files and historical agent results remain untouched. The live +EMT fixture now declares conditional exchangeability explicitly. Its results are +not identical inputs to the historical frozen comparison, and the README states +this boundary. The existing EMT action scorer is mechanism-specific; it is not +used as the oracle for the new external claim evaluation. + +## Limits + +The claim linter is not a general semantic/numeric verifier of arbitrary prose. +The external evaluation validates a narrow set of exact observed-mean claims, +using independent standard-library arithmetic, and rejects unsupported causal +or duplicate-grain claims. Its questions/oracle are authored in this project. +Conditional identification does not certify a causal effect's sign or magnitude. + +The paired agent experiment uses an actual mediated call to the same MCP handler, +not a simulated engine. It measures tool availability on fixed evidence summaries; +it is not a test of arbitrary autonomous exploration or native MCP transport. diff --git a/requirements/traceability.yaml b/requirements/traceability.yaml index c79b5bd..044272a 100644 --- a/requirements/traceability.yaml +++ b/requirements/traceability.yaml @@ -456,3 +456,23 @@ requirements: implementation: ["src/answerable/enterprise/connectors.py"] tests: ["tests/test_concrete_connectors.py"] status: verified + INV-001: + implementation: ["src/answerable/application/assessment_runner.py"] + tests: ["tests/test_validity_adversarial.py"] + status: implemented + INV-002: + implementation: ["src/answerable/evidence/verdict.py"] + tests: ["tests/test_validity_adversarial.py", "tests/test_external_evaluation.py"] + status: implemented + INV-008: + implementation: ["src/answerable/application/assessment_runner.py", "src/answerable/causal/contract.py"] + tests: ["tests/test_validity_adversarial.py"] + status: implemented + INV-009: + implementation: ["src/answerable/application/assessment_runner.py"] + tests: ["tests/test_validity_adversarial.py"] + status: implemented + INV-010: + implementation: ["src/answerable/application/assessment_runner.py"] + tests: ["tests/test_validity_adversarial.py"] + status: implemented diff --git a/schemas/v1/causal-contract.schema.json b/schemas/v1/causal-contract.schema.json index bb11d41..f8b77cb 100644 --- a/schemas/v1/causal-contract.schema.json +++ b/schemas/v1/causal-contract.schema.json @@ -4,16 +4,83 @@ "title": "Causal contract", "type": "object", "additionalProperties": false, - "required": ["treatment", "outcome", "population", "estimand", "strategy", "adjustment_set", "assumptions", "falsification_checks", "sensitivity_checks"], + "required": [ + "treatment", + "outcome", + "population", + "estimand", + "strategy", + "adjustment_set", + "assumptions", + "falsification_checks", + "sensitivity_checks" + ], "properties": { - "treatment": {"type": "string", "minLength": 1}, - "outcome": {"type": "string", "minLength": 1}, - "population": {"type": "string", "minLength": 1}, - "estimand": {"type": "string", "minLength": 1}, - "strategy": {"enum": ["randomized_experiment", "regression_adjustment", "matching_weighting", "difference_in_differences", "interrupted_time_series", "regression_discontinuity", "instrumental_variable", "synthetic_control", "panel_estimators"]}, - "adjustment_set": {"type": "array", "items": {"type": "string"}}, - "assumptions": {"type": "array", "items": {"type": "string"}}, - "falsification_checks": {"type": "array", "items": {"type": "string"}}, - "sensitivity_checks": {"type": "array", "items": {"type": "string"}} + "treatment": { + "type": "string", + "minLength": 1 + }, + "outcome": { + "type": "string", + "minLength": 1 + }, + "population": { + "type": "string", + "minLength": 1 + }, + "estimand": { + "type": "string", + "minLength": 1 + }, + "strategy": { + "enum": [ + "randomized_experiment", + "regression_adjustment", + "matching_weighting", + "difference_in_differences", + "interrupted_time_series", + "regression_discontinuity", + "instrumental_variable", + "synthetic_control", + "panel_estimators" + ] + }, + "adjustment_set": { + "type": "array", + "items": { + "type": "string" + } + }, + "assumptions": { + "type": "array", + "items": { + "type": "string" + } + }, + "falsification_checks": { + "type": "array", + "items": { + "type": "string" + } + }, + "sensitivity_checks": { + "type": "array", + "items": { + "type": "string" + } + }, + "identification_assumptions": { + "type": "array", + "items": { + "enum": [ + "conditional_exchangeability", + "random_assignment", + "parallel_trends", + "valid_instrument", + "valid_discontinuity" + ] + }, + "uniqueItems": true + } } } diff --git a/scripts/compare_external_agents.py b/scripts/compare_external_agents.py new file mode 100644 index 0000000..db2764b --- /dev/null +++ b/scripts/compare_external_agents.py @@ -0,0 +1,245 @@ +"""Paired text-agent experiment with an actual, mediated Answerable tool call. + +The same pinned model/command receives the same evidence in both arms. Only +tool availability changes. No raw rows, oracle labels or repo paths are sent. +The command reads a prompt from stdin and returns text on stdout; it must be +configured as a text-only model adapter, without ambient tools or memory. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import random +import re +import shutil +import subprocess +import tempfile +import time +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from statistics import mean + +from evaluate_external import digest, materialize, metrics + +from answerable.interfaces.mcp_handlers import assess_answerability + + +def build_context(folder: Path) -> dict[str, object]: + contract = json.loads((folder / "question.json").read_text()) + with (folder / "data.csv").open() as stream: + rows = list(csv.DictReader(stream)) + groups = sorted({r["group"] for r in rows}) + return { + "claim": contract["claims"][0], + "population": contract["population"], + "outcome": contract["outcome"], + "analysis_type": contract["analysis_type"], + "design": contract["causal"], + "time": contract["time"], + "row_count": len(rows), + "unique_entities": len({r["entity"] for r in rows}), + "group_counts": dict(Counter(r["group"] for r in rows)), + "group_means": { + g: mean(float(r["value"]) for r in rows if r["group"] == g) for g in groups + }, + } + + +def parse_decision(text: str) -> bool | None: + matches = re.findall(r"^DECISION: (ALLOW|BLOCK)$", text.strip(), flags=re.MULTILINE) + if len(matches) != 1 or text.strip().splitlines()[-1] != f"DECISION: {matches[0]}": + return None + return matches[0] == "ALLOW" + + +def run_case( + command: list[str], model: str, folder: Path, arm: str, timeout: float = 180 +) -> dict[str, object]: + context = build_context(folder) + prompt = ( + "Assess whether the exact claim is supported by the provided evidence. " + "Descriptive sample statements do not imply population inference or causality. " + "Do not use external files, tools or memory. Explain briefly, then end with exactly " + "DECISION: ALLOW or DECISION: BLOCK on its own line.\n" + + json.dumps(context, sort_keys=True) + + "\n" + ) + if arm == "tool": + prompt += ( + "A deterministic assessment tool is available. To call it, respond with exactly " + "CALL_ANSWERABLE. You will receive its real result, then make your own decision.\n" + ) + else: + prompt += "No assessment tool is available.\n" + transcript = [] + tool_calls = [] + start = time.perf_counter() + with tempfile.TemporaryDirectory(prefix="answerable-agent-") as scratch: + for turn in range(2): + try: + completed = subprocess.run( + command, + input=prompt, + text=True, + capture_output=True, + cwd=scratch, + timeout=timeout, + check=False, + ) + transcript.append( + { + "prompt": prompt, + "stdout": completed.stdout, + "stderr": completed.stderr, + "returncode": completed.returncode, + } + ) + if completed.returncode: + raise RuntimeError(f"adapter exited {completed.returncode}") + response = completed.stdout.strip() + if response == "CALL_ANSWERABLE" and arm == "tool" and turn == 0: + result = assess_answerability( + { + "data": str(folder / "data.csv"), + "question": str(folder / "question.json"), + "output": str(folder / "tool-assessment"), + } + ) + # Strip filesystem paths and identifiers; preserve substantive outputs. + public = { + key: result[key] + for key in ("verdict", "blockers", "allowed_claims", "forbidden_claims") + } + tool_calls.append({"tool": "assess_answerability", "result": public}) + prompt += ( + "\nAssistant: CALL_ANSWERABLE\nTool: " + + json.dumps(public) + + "\nThe tool call is complete. Return your final decision now.\n" + ) + continue + decision = parse_decision(response) + if decision is None: + raise ValueError("missing or ambiguous final decision") + return { + "allowed": decision, + "model": model, + "transcript": transcript, + "tool_calls": tool_calls, + "latency_ms": round((time.perf_counter() - start) * 1000), + } + except (OSError, subprocess.TimeoutExpired, RuntimeError, ValueError) as error: + return { + "allowed": None, + "model": model, + "error": str(error), + "transcript": transcript, + "tool_calls": tool_calls, + } + raise AssertionError("unreachable") + + +def paired_metrics(records: list[dict[str, object]]) -> dict[str, object]: + arms = {arm: metrics([r for r in records if r["arm"] == arm]) for arm in ("alone", "tool")} + arms["tool"]["tool_call_count"] = sum(len(r.get("tool_calls", [])) for r in records) + by_pair = {} + for row in records: + by_pair.setdefault((row["case_id"], row["repetition"]), {})[row["arm"]] = row + comparable = improved = regressed = 0 + for pair in by_pair.values(): + if len(pair) != 2 or any(row["allowed"] is None for row in pair.values()): + continue + comparable += 1 + before = pair["alone"]["allowed"] == pair["alone"]["expected_allow"] + after = pair["tool"]["allowed"] == pair["tool"]["expected_allow"] + improved += not before and after + regressed += before and not after + return { + "arms": arms, + "complete_pairs": comparable, + "improved": improved, + "regressed": regressed, + "accuracy_delta": (improved - regressed) / comparable if comparable else None, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--command-json", help="JSON argv array of a text-only model adapter") + parser.add_argument("--model", help="Exact provider model identifier (required for a real run)") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repetitions", type=int, default=2) + parser.add_argument("--seed", type=int, default=1729) + args = parser.parse_args() + if args.repetitions < 1: + parser.error("repetitions must be positive") + args.output = args.output.resolve() + if (args.output / "report.json").exists(): + parser.error("output already contains a report; use a new run directory") + cases = materialize(args.output) + command = json.loads(args.command_json) if args.command_json else [] + if command and (not isinstance(command, list) or not all(isinstance(x, str) for x in command)): + parser.error("command-json must be an argv array") + available = bool(command and shutil.which(command[0]) and args.model) + manifest = { + "protocol": "paired-external-v1", + "timestamp": datetime.now(UTC).isoformat(), + "model": args.model, + "command": command, + "repetitions": args.repetitions, + "seed": args.seed, + "case_count": len(cases), + "cases": cases, + "adapter_requirement": "text-only, no ambient tools or memory", + } + (args.output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + if not available: + report = { + "status": "not_run", + "reason": "No available text-model command and pinned model.", + "detected_cli": {name: shutil.which(name) for name in ("claude", "codex")}, + "expected_decisions": len(cases) * args.repetitions * 2, + "actual_decisions": 0, + "metrics": None, + "manifest_sha256": digest(args.output / "manifest.json"), + } + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + return 2 + records = [] + order = [(case, repetition) for case in cases for repetition in range(args.repetitions)] + random.Random(args.seed).shuffle(order) + with (args.output / "raw.jsonl").open("w") as log: + for index, (case, repetition) in enumerate(order): + # Counterbalance condition order; each invocation uses a fresh process/directory. + for arm in ("alone", "tool") if index % 2 == 0 else ("tool", "alone"): + folder = args.output / "cases" / case["case_id"] + record = { + "case_id": case["case_id"], + "repetition": repetition, + "arm": arm, + "expected_allow": case["expected_allow"], + "data_sha256": case["data_sha256"], + "question_sha256": case["question_sha256"], + **run_case(command, args.model, folder, arm), + } + records.append(record) + log.write(json.dumps(record) + "\n") + log.flush() + report = { + "status": "completed" if all(r["allowed"] is not None for r in records) else "incomplete", + "metrics": paired_metrics(records), + "actual_decisions": sum(r["allowed"] is not None for r in records), + "manifest_sha256": digest(args.output / "manifest.json"), + "raw_sha256": hashlib.sha256((args.output / "raw.jsonl").read_bytes()).hexdigest(), + } + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + return int(report["status"] != "completed") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_external.py b/scripts/evaluate_external.py new file mode 100644 index 0000000..f004346 --- /dev/null +++ b/scripts/evaluate_external.py @@ -0,0 +1,216 @@ +"""External-data smoke evaluation; claim labels are authored, not independent review.""" + +from __future__ import annotations + +import argparse +import calendar +import csv +import hashlib +import inspect +import json +import platform +from importlib.metadata import version +from pathlib import Path +from statistics import mean + +from answerable.application.assessment_runner import AssessmentRunner +from answerable.application.spec_loader import load_spec + +ROOT = Path(__file__).resolve().parents[1] +SUITE = ROOT / "benchmarks/external" + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def materialize(output: Path) -> list[dict[str, object]]: + """Transform external dates without inventing observations; preserve every value.""" + manifest = json.loads((SUITE / "manifest.json").read_text()) + cases = [] + for source in manifest["sources"]: + path = SUITE / source["path"] + if digest(path) != source["sha256"]: + raise ValueError(f"source hash mismatch: {path}") + original = list(csv.DictReader(path.open(encoding="utf-8"))) + rows = [] + for row in original: + if source["id"] == "flights": + month = list(calendar.month_name).index(row["month"]) + day = f"{row['year']}-{month:02d}-01" + value = float(row["passengers"]) + else: + day = row["Date"] + month = int(day[5:7]) + value = float(row["Temp"]) + rows.append( + { + "entity": day, + "date": day + "T00:00:00+00:00", + "group": int(month > 6), + "value": value, + } + ) + for scope in ("full", "first_year"): + selected = ( + rows + if scope == "full" + else [r for r in rows if r["entity"][:4] == rows[0]["entity"][:4]] + ) + a = mean(r["value"] for r in selected if r["group"] == 0) + b = mean(r["value"] for r in selected if r["group"] == 1) + for variant in ("observed", "causal", "duplicate"): + case_id = f"{source['id']}-{scope}-{variant}" + folder = output / "cases" / case_id + folder.mkdir(parents=True, exist_ok=True) + data = [*selected, selected[0]] if variant == "duplicate" else selected + with (folder / "data.csv").open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=["entity", "date", "group", "value"]) + writer.writeheader() + writer.writerows(data) + claim = ( + f"Among these recorded {source['unit']}s, the observed mean was {a:.6g} " + f"in January-June and {b:.6g} in July-December ({source['units']})." + ) + if variant == "causal": + claim = ( + "Assignment to July-December caused the observed difference in outcomes." + ) + kind = "causal" if variant == "causal" else "descriptive" + question = { + "question_id": case_id, + "raw_question": claim, + "normalized_question": claim, + "language": "en", + "analysis_type": kind, + "unit_of_analysis": source["unit"], + "population": { + "description": f"{source['id']} observations ({scope}); no extrapolation" + }, + "outcome": { + "metric_id": "value", + "definition": source["units"], + "value_type": "continuous", + }, + "time": { + "observation_start": selected[0]["date"], + "observation_end": "2000-01-01T00:00:00+00:00", + }, + "data": { + "entity_column": "entity", + "event_time_column": "date", + "treatment_column": "group", + "outcome_column": "value", + "observation_window_days": 1, + "analysis_end": "2000-01-01T00:00:00+00:00", + }, + "causal": { + "treatment": "group", + "outcome": "value", + "population": "recorded observations", + "estimand": "ATE", + "strategy": "regression_adjustment", + }, + "claims": [{"text": claim, "claim_class": kind}], + } + (folder / "question.json").write_text(json.dumps(question, indent=2) + "\n") + cases.append( + { + "case_id": case_id, + "source": source["id"], + "variant": variant, + "expected_allow": variant == "observed", + "claim": claim, + "data_sha256": digest(folder / "data.csv"), + "question_sha256": digest(folder / "question.json"), + "expected_means": {"0": a, "1": b}, + } + ) + return cases + + +def metrics(records: list[dict[str, object]]) -> dict[str, object]: + valid = [r for r in records if r["expected_allow"]] + invalid = [r for r in records if not r["expected_allow"]] + false_blocks = sum(r["allowed"] is False for r in valid) + unsafe = sum(r["allowed"] is True for r in invalid) + return { + "total": len(records), + "valid_claims": len(valid), + "invalid_claims": len(invalid), + "false_blocks": false_blocks, + "unsafe_allows": unsafe, + "false_block_rate": false_blocks / len(valid) if valid else None, + "unsafe_allow_rate": unsafe / len(invalid) if invalid else None, + "release_false_blocks": sum(r.get("release_allowed") is False for r in valid), + "errors": sum(r.get("error") is not None for r in records), + } + + +def engine_fingerprint() -> str: + root = Path(inspect.getfile(AssessmentRunner)).parents[1] + entries = [(str(path.relative_to(root)), digest(path)) for path in sorted(root.rglob("*.py"))] + return hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() + + +def evaluate(output: Path) -> dict[str, object]: + output.mkdir(parents=True, exist_ok=True) + cases = materialize(output) + # Freeze labels and hashes before invoking the engine; the oracle is not a verdict lookup. + (output / "cases.json").write_text(json.dumps(cases, indent=2) + "\n") + records = [] + for case in cases: + folder = output / "cases" / str(case["case_id"]) + try: + run = AssessmentRunner().run( + data_sources=(folder / "data.csv",), + spec=load_spec(folder / "question.json"), + output_directory=folder / "assessment", + ) + records.append( + { + **case, + "allowed": case["claim"] in run.allowed_claims, + "verdict": run.verdict.value, + "release_allowed": case["claim"] in run.allowed_claims + and run.verdict.value in {"ANSWERABLE", "ANSWERABLE_WITH_ASSUMPTIONS"}, + "blockers": [b.finding_id for b in run.blockers], + "observed_means": { + r["group"]: r["rate"] for r in run.observations["outcome_rates"] + }, + } + ) + except Exception as error: + records.append({**case, "allowed": None, "error": f"{type(error).__name__}: {error}"}) + report = { + "protocol": "external-smoke-v1", + "engine_sha256": engine_fingerprint(), + "evaluator_sha256": digest(Path(__file__)), + "dependencies": {name: version(name) for name in ("duckdb", "pyyaml", "sqlglot")}, + "python": platform.python_version(), + "cases_sha256": digest(output / "cases.json"), + "metrics": metrics(records), + "limitations": [ + "Two external datasets; questions and labels authored within this project.", + "Duplicate cases are explicit perturbations of external data.", + "No population/causal generalization or independent expert validation.", + "Cases within a source are correlated; no independence-based significance test.", + ], + "records": records, + } + (output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + report = evaluate(args.output) + print(json.dumps(report["metrics"], indent=2)) + scores = report["metrics"] + return int(bool(scores["false_blocks"] or scores["unsafe_allows"] or scores["errors"])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/answerable/application/assessment_runner.py b/src/answerable/application/assessment_runner.py index e709a8d..5539fa4 100644 --- a/src/answerable/application/assessment_runner.py +++ b/src/answerable/application/assessment_runner.py @@ -11,9 +11,9 @@ from answerable.analysis.grain import GrainAnalyzer, GrainStatus from answerable.application.models import AssessmentRun, AssessmentSpec, DataMapping from answerable.causal.contract import CausalContract, CausalIdentifier -from answerable.domain.models import CheckPlan, CheckSpec, Verdict +from answerable.domain.models import AnalysisType, CheckPlan, CheckSpec, Verdict from answerable.domain.serialization import fingerprint, to_dict -from answerable.evidence.claims import ClaimContext +from answerable.evidence.claims import ClaimClass, ClaimContext from answerable.evidence.graph import ( EdgeType, EvidenceGraphStore, @@ -42,6 +42,10 @@ _CATEGORY = { "causal_identification_failure": "identification", + "identification_evidence_missing": "missing_evidence", + "identification_assumed": "assumption", + "missing_sensitivity": "assumption", + "adjustment_mapping_mismatch": "missing_evidence", "positivity_violation": "identification", "immature_cohort": "missing_evidence", "right_censoring": "missing_evidence", @@ -50,7 +54,7 @@ "prediction_leakage": "data_integrity", "duplicate_entities": "data_integrity", "ambiguous_grain": "data_integrity", - "insufficient_power": "missing_evidence", + "insufficient_power": "power", "definition_change": "data_integrity", "informative_missingness": "missing_evidence", } @@ -66,6 +70,18 @@ ) _DESIGN_IMPOSSIBLE = frozenset({"causal_identification_failure", "positivity_violation"}) _REPAIR = { + "identification_evidence_missing": ( + "Explicit design evidence or declared identifying assumptions.", + "Empirical overlap alone does not establish causal identification.", + True, + "document the design and review identifying assumptions", + ), + "adjustment_mapping_mismatch": ( + "A checked covariate mapping matching the declared adjustment set.", + "Support was not checked over the intended adjustment variables.", + True, + "reconcile the adjustment set with the input column mapping", + ), "causal_identification_failure": ( "A comparison population that identifies the requested causal estimand.", "Without it no observed difference can be attributed to the campaign.", @@ -161,6 +177,125 @@ ) +_FINDING_CHECK = { + "duplicate_entities": "chk_grain_uniqueness", + "ambiguous_grain": "chk_grain_uniqueness", + "invalid_event_time": "chk_temporal_maturity", + "timezone_ambiguity": "chk_temporal_maturity", + "immature_cohort": "chk_temporal_maturity", + "right_censoring": "chk_temporal_maturity", + "prediction_leakage": "chk_temporal_maturity", + "definition_change": "chk_metric_definition", + "informative_missingness": "chk_outcome_missingness", + "insufficient_power": "chk_statistical_power", + "positivity_violation": "chk_positivity_overlap", + "causal_identification_failure": "chk_causal_identification", + "identification_evidence_missing": "chk_causal_identification", + "identification_assumed": "chk_causal_identification", + "adjustment_mapping_mismatch": "chk_causal_identification", + "missing_sensitivity": "chk_causal_identification", +} +_EXTRA_CHECKS = tuple( + CheckSpec( + check_id=check_id, + check_type=kind, + check_version="1.0", + requirement_id=requirement, + executor="python", + severity_on_failure="blocker", + rationale=rationale, + mandatory=True, + ) + for check_id, kind, requirement, rationale in ( + ( + "chk_metric_definition", + "metric.definition", + "INV-009", + "Check stable metric definitions.", + ), + ( + "chk_outcome_missingness", + "quality.missingness", + "INV-009", + "Compare outcome missingness by group.", + ), + ( + "chk_statistical_power", + "statistical.power", + "INV-005", + "Assess inferential precision limitations.", + ), + ("chk_observed_means", "descriptive.means", "INV-001", "Compute observed group means."), + ) +) + + +def _requires_causal(spec: AssessmentSpec) -> bool: + return spec.contract.analysis_type is AnalysisType.CAUSAL or any( + claim.claim_class is ClaimClass.CAUSAL for claim in spec.claims + ) + + +def _checks_for(spec: AssessmentSpec) -> tuple[CheckSpec, ...]: + checks = list(_CHECKS[:2]) + if _requires_causal(spec): + checks.extend(_CHECKS[2:]) + for check in _EXTRA_CHECKS: + if check.check_id == "chk_metric_definition" and not spec.mapping.metric_definition_column: + continue + if ( + check.check_id == "chk_statistical_power" + and spec.contract.analysis_type is AnalysisType.DESCRIPTIVE + and not _requires_causal(spec) + ): + continue + checks.append(check) + return tuple(checks) + + +def _evidence_status( + spec: AssessmentSpec, rows: tuple[dict[str, object], ...], findings: tuple[Finding, ...] +) -> dict[str, Any]: + facts: list[dict[str, object]] = [ + {"id": "row_count", "value": len(rows), "check_id": "chk_grain_uniqueness"}, + { + "id": "unique_entities", + "value": len({r["entity"] for r in rows}) == len(rows), + "check_id": "chk_grain_uniqueness", + }, + ] + if _requires_causal(spec): + facts.append( + { + "id": "empirical_stratum_overlap", + "value": _overlap(rows, spec.mapping), + "check_id": "chk_positivity_overlap", + } + ) + assumptions = list( + dict.fromkeys( + ( + *spec.contract.assumptions, + *spec.causal.assumptions, + *spec.causal.identification_assumptions, + ) + ) + ) + return { + "verified_facts": facts, + "declared_assumptions": assumptions, + "unverifiable_conditions": ( + [ + "Absence of unmeasured confounding is not established by observed overlap.", + "The declared design assumptions require evidence beyond this table.", + ] + if _requires_causal(spec) + else [] + ), + "conditional_identification": any(f.code == "identification_assumed" for f in findings), + } + + @dataclass(frozen=True, slots=True) class _Descriptive: """Group rates for the declared outcome. Descriptive only, never causal.""" @@ -205,7 +340,7 @@ def run( "asm_" + fingerprint( { - "question": to_dict(spec.contract), + "spec": to_dict(spec), "sources": [item.fingerprint for item in snapshots], } )[:16] @@ -217,8 +352,10 @@ def run( findings = _run_checks(snapshots[0], rows, descriptive, spec) finding_inputs = tuple(_to_finding_input(item) for item in findings) - identified = not any( - item.code in _DESIGN_IMPOSSIBLE + identified = _requires_causal(spec) and not any( + item.code + in _DESIGN_IMPOSSIBLE + | {"identification_evidence_missing", "adjustment_mapping_mismatch"} for item in findings if item.severity is Severity.BLOCKER ) @@ -241,7 +378,8 @@ def run( verdict = VerdictEngine().decide(finding_inputs, claims=claims) repairs = _repair_plan(findings) - graph = _build_graph(spec, snapshots, descriptive, findings, verdict) + status = _evidence_status(spec, rows, findings) + graph = _build_graph(spec, snapshots, descriptive, findings, verdict, status) observations = { "identified": identified, "outcome_rates": [ @@ -250,6 +388,7 @@ def run( ], "observed_difference": descriptive.difference, "baseline": descriptive.baseline, + "evidence_status": status, } payload = _warrant_payload( assessment_id, spec, snapshots, verdict, findings, repairs, observations, graph @@ -371,12 +510,23 @@ def _describe(path: Path, mapping: DataMapping) -> _Descriptive: def _overlap(rows: tuple[dict[str, object], ...], mapping: DataMapping) -> bool: - """Positivity: some stratum must contain both treatment levels.""" + """Conservative binary support check over every declared target stratum. + + Empirical support is not proof of population positivity or exchangeability. + Continuous covariates require an explicitly reviewed coarsening upstream. + """ strata: dict[tuple[object, ...], set[object]] = {} for row in rows: key = tuple(row.get(name) for name in mapping.covariate_columns) strata.setdefault(key, set()).add(row["treatment"]) - return any(len(levels) > 1 for levels in strata.values()) + levels = {row["treatment"] for row in rows} + return ( + len(levels) == 2 + and None not in levels + and bool(strata) + and all(values == levels for values in strata.values()) + and all(None not in key for key in strata) + ) def _run_checks( @@ -432,27 +582,59 @@ def _run_checks( findings.extend(TemporalAssessor.definition_changes(versions)) findings.extend(_missingness_findings(rows, spec.mapping)) - overlap = _overlap(rows, spec.mapping) - if not overlap: - findings.append( - Finding( - "positivity_violation", - Severity.BLOCKER, - "No covariate stratum contains both exposed and unexposed entities.", - (spec.mapping.treatment_column, *spec.mapping.covariate_columns), + if _requires_causal(spec): + overlap = _overlap(rows, spec.mapping) + if not overlap: + findings.append( + Finding( + "positivity_violation", + Severity.BLOCKER, + "Every target covariate stratum must contain both binary treatment levels.", + (spec.mapping.treatment_column, *spec.mapping.covariate_columns), + ) ) - ) - findings.extend( - CausalIdentifier() - .assess( + mapping_matches = spec.causal.adjustment_set == frozenset(spec.mapping.covariate_columns) + if not mapping_matches: + findings.append( + Finding( + "adjustment_mapping_mismatch", + Severity.BLOCKER, + "Declared adjustment set differs from the checked covariates.", + ) + ) + declared = set(spec.causal.identification_assumptions) + causal = CausalIdentifier().assess( spec.causal, _ObservedDifferenceEstimator(descriptive), positivity_supported=overlap, - exchangeability_supported=overlap, + exchangeability_supported="conditional_exchangeability" in declared and mapping_matches, + randomization_valid="random_assignment" in declared, + parallel_trends_supported="parallel_trends" in declared, + instrument_valid="valid_instrument" in declared, + discontinuity_valid="valid_discontinuity" in declared, ) - .findings - ) - findings.extend(_power_findings(rows)) + for finding in causal.findings: + if finding.code == "causal_identification_failure" and overlap: + findings.append( + Finding( + "identification_evidence_missing", + Severity.BLOCKER, + "Identifying conditions are neither established nor explicitly assumed.", + ) + ) + else: + findings.append(finding) + if causal.identified: + findings.append( + Finding( + "identification_assumed", + Severity.WARNING, + "Identification is conditional on declared assumptions. " + "Not verified: the mean difference remains descriptive, not adjusted.", + ) + ) + if spec.contract.analysis_type is not AnalysisType.DESCRIPTIVE or _requires_causal(spec): + findings.extend(_power_findings(rows)) return tuple(findings) @@ -558,6 +740,7 @@ def _build_graph( descriptive: _Descriptive, findings: tuple[Finding, ...], verdict: VerdictResult, + status: dict[str, Any], ) -> dict[str, object]: store = EvidenceGraphStore() store.add_node( @@ -579,10 +762,9 @@ def _build_graph( }, ) ) - for check in _CHECKS: + for check in _checks_for(spec): store.add_node(GraphNode(check.check_id, NodeType.EXECUTION, {"type": check.check_type})) - for snapshot in snapshots: - store.add_edge(GraphEdge(check.check_id, snapshot.asset_id, EdgeType.USES)) + store.add_edge(GraphEdge(check.check_id, snapshots[0].asset_id, EdgeType.USES)) store.add_node( GraphNode( "obs_outcome_rates", @@ -595,24 +777,37 @@ def _build_graph( }, ) ) - store.add_edge(GraphEdge("obs_outcome_rates", "chk_grain_uniqueness", EdgeType.COMPUTED_FROM)) + store.add_edge(GraphEdge("obs_outcome_rates", "chk_observed_means", EdgeType.COMPUTED_FROM)) for finding in findings: node_type = NodeType.BLOCKER if finding.severity is Severity.BLOCKER else NodeType.WARNING store.add_node( GraphNode(f"finding_{finding.code}", node_type, {"message": finding.message}) ) store.add_edge( - GraphEdge(f"finding_{finding.code}", "chk_positivity_overlap", EdgeType.DEPENDS_ON) + GraphEdge(f"finding_{finding.code}", _FINDING_CHECK[finding.code], EdgeType.DEPENDS_ON) ) + for index, fact in enumerate(status["verified_facts"]): + node_id = f"fact_{index}" + store.add_node(GraphNode(node_id, NodeType.FACT, fact)) + store.add_edge(GraphEdge(node_id, str(fact["check_id"]), EdgeType.COMPUTED_FROM)) + for key, node_type in ( + ("declared_assumptions", NodeType.ASSUMPTION), + ("unverifiable_conditions", NodeType.UNVERIFIABLE_CONDITION), + ): + for index, statement in enumerate(status[key]): + store.add_node(GraphNode(f"{key}_{index}", node_type, {"statement": statement})) for index, claim in enumerate(verdict.allowed_claims): node = f"claim_allowed_{index}" store.add_node(GraphNode(node, NodeType.ALLOWED_CLAIM, {"text": claim})) store.add_edge(GraphEdge(node, "obs_outcome_rates", EdgeType.DEPENDS_ON)) + for index in range(len(status["declared_assumptions"])): + store.add_edge(GraphEdge(node, f"declared_assumptions_{index}", EdgeType.ASSUMES)) for index, claim in enumerate(verdict.forbidden_claims): node = f"claim_forbidden_{index}" store.add_node(GraphNode(node, NodeType.FORBIDDEN_CLAIM, {"text": claim})) for finding in findings: - if finding.severity is Severity.BLOCKER: + candidate = next(candidate for candidate in spec.claims if candidate.text == claim) + if VerdictEngine.blocks_claim(_to_finding_input(finding), candidate.claim_class): store.add_edge(GraphEdge(f"finding_{finding.code}", node, EdgeType.BLOCKS)) store.validate_claims() return store.export() @@ -645,7 +840,15 @@ def _warrant_payload( {"code": item.finding_id, "category": item.category, "message": item.message} for item in verdict.decisive_findings ], - "assumptions": list(spec.causal.assumptions), + "assumptions": list( + dict.fromkeys( + ( + *spec.contract.assumptions, + *spec.causal.assumptions, + *spec.causal.identification_assumptions, + ) + ) + ), "limitations": [item.message for item in findings if item.severity is Severity.WARNING], "data_quality_relevance": observations, "minimum_evidence_plan": { @@ -653,9 +856,9 @@ def _warrant_payload( "minimal": to_dict(minimal) if minimal else None, }, "permitted_analysis": ( - "descriptive comparison only" - if verdict.verdict is not Verdict.ANSWERABLE - else "full analysis" + "only the listed allowed claims; no adjusted causal estimate is computed" + if verdict.allowed_claims + else "none" ), "provenance": [ { @@ -667,7 +870,7 @@ def _warrant_payload( for item in snapshots ], "reproducibility_manifest": { - "checks": [to_dict(item) for item in _CHECKS], + "checks": [to_dict(item) for item in _checks_for(spec)], "evidence_graph_hash": graph["content_hash"], }, "approvals": [], @@ -677,6 +880,10 @@ def _warrant_payload( def _explain(verdict: Verdict, findings: tuple[Finding, ...]) -> str: blockers = [item.message for item in findings if item.severity is Severity.BLOCKER] + if verdict is Verdict.ANSWERABLE_WITH_ASSUMPTIONS: + return ( + "The conclusion is conditional on declared assumptions; these are not verified facts." + ) if not blockers: return "The available evidence supports the requested conclusion." return ( @@ -705,7 +912,7 @@ def _write( CheckPlan( plan_id=f"pln_{assessment_id.removeprefix('asm_')}", assessment_id=assessment_id, - checks=_CHECKS, + checks=_checks_for(spec), ) ), "findings": [to_dict(item) for item in findings], diff --git a/src/answerable/application/spec_loader.py b/src/answerable/application/spec_loader.py index b437397..3829f59 100644 --- a/src/answerable/application/spec_loader.py +++ b/src/answerable/application/spec_loader.py @@ -91,6 +91,7 @@ def load_spec(path: Path) -> AssessmentSpec: assumptions=tuple(causal.get("assumptions", ())), falsification_checks=tuple(causal.get("falsification_checks", ())), sensitivity_checks=tuple(causal.get("sensitivity_checks", ())), + identification_assumptions=tuple(causal.get("identification_assumptions", ())), ), claims=tuple( ClaimCandidate(text=item["text"], claim_class=ClaimClass(item["claim_class"])) diff --git a/src/answerable/causal/contract.py b/src/answerable/causal/contract.py index 44e215b..7914ac3 100644 --- a/src/answerable/causal/contract.py +++ b/src/answerable/causal/contract.py @@ -30,6 +30,7 @@ class CausalContract: assumptions: tuple[str, ...] = () falsification_checks: tuple[str, ...] = () sensitivity_checks: tuple[str, ...] = () + identification_assumptions: tuple[str, ...] = () def __post_init__(self) -> None: if not all((self.treatment, self.outcome, self.population, self.estimand)): @@ -38,6 +39,15 @@ def __post_init__(self) -> None: ) if self.treatment == self.outcome or self.treatment in self.adjustment_set: raise ValueError("invalid treatment/outcome/adjustment relationship") + known = { + "conditional_exchangeability", + "random_assignment", + "parallel_trends", + "valid_instrument", + "valid_discontinuity", + } + if set(self.identification_assumptions) - known: + raise ValueError("unknown identification assumption") class Estimator(Protocol): diff --git a/src/answerable/domain/serialization.py b/src/answerable/domain/serialization.py index 494acbc..80fdfad 100644 --- a/src/answerable/domain/serialization.py +++ b/src/answerable/domain/serialization.py @@ -22,6 +22,10 @@ def to_dict(value: object) -> Any: return value.isoformat() if isinstance(value, tuple): return [to_dict(item) for item in value] + if isinstance(value, frozenset): + return sorted( + (to_dict(item) for item in value), key=lambda item: json.dumps(item, sort_keys=True) + ) if isinstance(value, list): return [to_dict(item) for item in value] if isinstance(value, dict): @@ -65,6 +69,10 @@ def _decode(annotation: object, value: object) -> Any: if not isinstance(value, list): raise TypeError("tuple value must be an array") return tuple(_decode(item_type, item) for item in value) + if origin is frozenset: + if not isinstance(value, list): + raise TypeError("frozenset value must be an array") + return frozenset(_decode(args[0], item) for item in value) if origin is dict: if not isinstance(value, dict): raise TypeError("dict value must be an object") diff --git a/src/answerable/evidence/claims.py b/src/answerable/evidence/claims.py index 5e274a8..402f8f3 100644 --- a/src/answerable/evidence/claims.py +++ b/src/answerable/evidence/claims.py @@ -30,6 +30,8 @@ class ClaimLinter: def lint(self, claim: str, context: ClaimContext) -> tuple[str, ...]: violations: list[str] = [] + if context.claim_class is ClaimClass.CAUSAL and not context.causal_gate: + violations.append("causal_class_without_identification") if self._CAUSAL.search(claim) and ( context.claim_class is not ClaimClass.CAUSAL or not context.causal_gate ): diff --git a/src/answerable/evidence/graph.py b/src/answerable/evidence/graph.py index 38e6297..fd81c33 100644 --- a/src/answerable/evidence/graph.py +++ b/src/answerable/evidence/graph.py @@ -13,6 +13,7 @@ class NodeType(StrEnum): OBSERVATION = "observation" FACT = "fact" ASSUMPTION = "assumption" + UNVERIFIABLE_CONDITION = "unverifiable_condition" INFERENCE = "inference" BLOCKER = "blocker" WARNING = "warning" diff --git a/src/answerable/evidence/verdict.py b/src/answerable/evidence/verdict.py index 805e181..f998ac1 100644 --- a/src/answerable/evidence/verdict.py +++ b/src/answerable/evidence/verdict.py @@ -4,7 +4,7 @@ from enum import StrEnum from answerable.domain.models import Verdict -from answerable.evidence.claims import ClaimContext, ClaimLinter +from answerable.evidence.claims import ClaimClass, ClaimContext, ClaimLinter class Repairability(StrEnum): @@ -46,12 +46,31 @@ def decide( forbidden: list[str] = [] linter = ClaimLinter() for claim, context in claims: - if linter.lint(claim, context): + if linter.lint(claim, context) or any( + self.blocks_claim(item, context.claim_class) for item in blockers + ): forbidden.append(claim) else: allowed.append(claim) return VerdictResult(verdict, blockers or findings, tuple(allowed), tuple(forbidden)) + @staticmethod + def blocks_claim(finding: FindingInput, claim_class: ClaimClass) -> bool: + """Scope blockers to claims, independently of their surface wording.""" + if finding.severity not in {"blocker", "fatal"}: + return False + if finding.affects_all_claims or finding.category in { + "execution_fatal", + "schema_fatal", + "security_fatal", + "data_integrity", + "misleading_question", + }: + return True + # A raw observed association may survive a missing causal prerequisite. + # It does not establish a population effect, prediction or recommendation. + return claim_class not in {ClaimClass.DESCRIPTIVE, ClaimClass.ASSOCIATION} + @staticmethod def _precedence(findings: tuple[FindingInput, ...]) -> Verdict: categories = {item.category for item in findings if item.severity in {"blocker", "fatal"}} diff --git a/src/answerable/mutation_benchmark.py b/src/answerable/mutation_benchmark.py index f8d45d5..1be9f13 100644 --- a/src/answerable/mutation_benchmark.py +++ b/src/answerable/mutation_benchmark.py @@ -320,6 +320,7 @@ def _question_yaml(scenario: Scenario) -> str: estimand: "Average treatment effect of exposure on {metric_id}" strategy: regression_adjustment adjustment_set: ["channel"] + identification_assumptions: ["conditional_exchangeability"] assumptions: - "Exposure is recorded without error." falsification_checks: [] diff --git a/src/answerable/reports/markdown.py b/src/answerable/reports/markdown.py index 3297880..bd194c0 100644 --- a/src/answerable/reports/markdown.py +++ b/src/answerable/reports/markdown.py @@ -16,10 +16,13 @@ def render_markdown(run: AssessmentRun) -> str: question = data["question"] outcome_rates: Any = run.observations.get("outcome_rates", []) rates = [ - f"{item['group']}: {item['rate']:.1%} of {item['entities']} {question['unit_of_analysis']}s" + f"{item['group']}: observed mean {item['rate']:.6g}; " + f"{item['entities']} {question['unit_of_analysis']}s" for item in outcome_rates ] repairs = data["minimum_evidence_plan"]["minimal"] + status = run.observations.get("evidence_status", {}) + status = status if isinstance(status, dict) else {} sections = [ "# Evidence Warrant", "", @@ -55,9 +58,21 @@ def render_markdown(run: AssessmentRun) -> str: f"- Why it matters: {repairs['why_it_matters']}\n" f"- How to get it: {repairs['collection_method']}" if repairs - else "- Nothing; the evidence is complete." + else "- No repairable blocker recorded; review the assumptions below." ), "", + "## Verified facts", + "", + _bullets(status.get("verified_facts", []), "No facts recorded."), + "", + "## Declared assumptions (not verified)", + "", + _bullets(status.get("declared_assumptions", []), "None declared."), + "", + "## Conditions not verifiable from this table", + "", + _bullets(status.get("unverifiable_conditions", []), "None required for this scope."), + "", "## Provenance", "", _bullets( diff --git a/tests/e2e/test_runner_edge_cases.py b/tests/e2e/test_runner_edge_cases.py index 66039ee..b21f2d7 100644 --- a/tests/e2e/test_runner_edge_cases.py +++ b/tests/e2e/test_runner_edge_cases.py @@ -2,6 +2,7 @@ import json import unittest +from dataclasses import replace from datetime import UTC, datetime from pathlib import Path from tempfile import TemporaryDirectory @@ -85,18 +86,25 @@ def test_supported_design_opens_the_causal_gate(self) -> None: # "cleanly answerable" case has to actually be powered, not # just directionally correct on a handful of rows. root = _case(Path(directory), "2025-01-10T00:00:00+00:00", size=160) + spec = load_spec(root / "question.json") + spec = replace( + spec, + causal=replace( + spec.causal, identification_assumptions=("conditional_exchangeability",) + ), + ) run = AssessmentRunner(signer="ci", secret=b"secret").run( data_sources=(root / "customers.csv",), - spec=load_spec(root / "question.json"), + spec=spec, output_directory=root / "out", ) - self.assertEqual(run.verdict, Verdict.ANSWERABLE) + self.assertEqual(run.verdict, Verdict.ANSWERABLE_WITH_ASSUMPTIONS) self.assertEqual(run.blockers, ()) self.assertEqual(run.forbidden_claims, ()) self.assertEqual(run.allowed_claims, ("Exposure caused higher 90-day retention.",)) markdown = (root / "out" / "warrant.md").read_text(encoding="utf-8") self.assertIn("Nothing blocks this question.", markdown) - self.assertIn("the evidence is complete", markdown) + self.assertIn("Declared assumptions (not verified)", markdown) def test_timezone_naive_timestamps_are_a_data_integrity_failure(self) -> None: with TemporaryDirectory() as directory: diff --git a/tests/test_external_evaluation.py b/tests/test_external_evaluation.py new file mode 100644 index 0000000..2cb6c5b --- /dev/null +++ b/tests/test_external_evaluation.py @@ -0,0 +1,108 @@ +"""INV-002 / BENCH: retain valid claims and audit real mediated tool calls.""" + +import json +import sys +from pathlib import Path +from subprocess import CompletedProcess, TimeoutExpired +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from compare_external_agents import build_context, paired_metrics, parse_decision, run_case +from evaluate_external import evaluate, materialize, metrics + + +def test_BENCH_external_data_keeps_valid_claims_and_rejects_invalid(tmp_path: Path) -> None: + report = evaluate(tmp_path) + assert report["metrics"] == { + "total": 12, + "valid_claims": 4, + "invalid_claims": 8, + "false_blocks": 0, + "unsafe_allows": 0, + "false_block_rate": 0.0, + "unsafe_allow_rate": 0.0, + "errors": 0, + "release_false_blocks": 0, + } + for row in report["records"]: + if row["variant"] == "observed": + assert row["expected_means"] == pytest.approx(row["observed_means"]) + + +def test_BENCH_failure_metrics_do_not_hide_unparseable_answers() -> None: + assert metrics([])["false_block_rate"] is None + records = [ + {"expected_allow": True, "allowed": False}, + {"expected_allow": False, "allowed": True}, + {"expected_allow": True, "allowed": None, "error": "timeout"}, + ] + assert metrics(records)["false_blocks"] == 1 + assert metrics(records)["unsafe_allows"] == 1 + assert metrics(records)["errors"] == 1 + assert paired_metrics([])["accuracy_delta"] is None + + +def test_BENCH_context_excludes_oracle_and_tool_arm_executes_real_handler(tmp_path: Path) -> None: + case = materialize(tmp_path)[1] + folder = tmp_path / "cases" / case["case_id"] + assert "expected_allow" not in json.dumps(build_context(folder)) + replies = [ + CompletedProcess([], 0, "CALL_ANSWERABLE", ""), + CompletedProcess([], 0, "Evidence is missing.\nDECISION: BLOCK", ""), + ] + with patch("compare_external_agents.subprocess.run", side_effect=replies): + result = run_case(["test-only-adapter"], "test-double", folder, "tool") + assert result["allowed"] is False + assert len(result["tool_calls"]) == 1 + assert (folder / "tool-assessment/warrant.json").is_file() + assert "identification_evidence_missing" in json.dumps(result["tool_calls"]) + assert "Tool:" in result["transcript"][1]["prompt"] + + +@pytest.mark.parametrize( + "response", ["DECISION: ALLOW\nDECISION: BLOCK", "BLOCK", "DECISION: ALLOW\nextra"] +) +def test_BENCH_ambiguous_outputs_are_errors(response: str) -> None: + assert parse_decision(response) is None + + +def test_BENCH_alone_cannot_request_tool_and_adapter_errors_are_recorded(tmp_path: Path) -> None: + case = materialize(tmp_path)[0] + folder = tmp_path / "cases" / case["case_id"] + for response in ( + CompletedProcess([], 0, "CALL_ANSWERABLE", ""), + CompletedProcess([], 1, "DECISION: ALLOW", "error"), + ): + with patch("compare_external_agents.subprocess.run", return_value=response): + result = run_case(["test-only-adapter"], "test-double", folder, "alone") + assert result["allowed"] is None + assert result["error"] + assert not result["tool_calls"] + with patch("compare_external_agents.subprocess.run", side_effect=TimeoutExpired("adapter", 1)): + assert run_case(["test-only-adapter"], "test-double", folder, "tool")["allowed"] is None + + +def test_BENCH_paired_scores_include_regressions_and_exclude_incomplete_pairs() -> None: + records = [] + for case, expected, before, after in [ + ("improves", True, False, True), + ("regresses", False, False, True), + ("missing", True, None, True), + ]: + for arm, answer in [("alone", before), ("tool", after)]: + records.append( + { + "case_id": case, + "repetition": 0, + "arm": arm, + "expected_allow": expected, + "allowed": answer, + "tool_calls": [], + } + ) + result = paired_metrics(records) + assert result["complete_pairs"] == 2 + assert result["improved"] == result["regressed"] == 1 + assert result["accuracy_delta"] == 0 diff --git a/tests/test_mutation_benchmark.py b/tests/test_mutation_benchmark.py index 2731aa7..7f03754 100644 --- a/tests/test_mutation_benchmark.py +++ b/tests/test_mutation_benchmark.py @@ -55,7 +55,9 @@ def test_mutation_benchmark_executes_runner_and_passes_release_gate(tmp_path: Pa assert report.release_pass assert len(report.reproducibility_hash) == 64 assert (tmp_path / "bench" / "mutation_report.json").is_file() - assert all(item.baseline_verdict == "ANSWERABLE" for item in report.observations) + assert all( + item.baseline_verdict == "ANSWERABLE_WITH_ASSUMPTIONS" for item in report.observations + ) invalidated = [ item for item in report.observations diff --git a/tests/test_validity_adversarial.py b/tests/test_validity_adversarial.py new file mode 100644 index 0000000..0d1813d --- /dev/null +++ b/tests/test_validity_adversarial.py @@ -0,0 +1,217 @@ +"""Counterexamples to INV-001/002/008/010, independent of EMT's oracle.""" + +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from answerable.application.assessment_runner import AssessmentRunner, _overlap +from answerable.application.models import ClaimCandidate +from answerable.application.spec_loader import load_spec +from answerable.causal.contract import CausalContract, IdentificationStrategy +from answerable.domain.models import AnalysisType, Verdict +from answerable.domain.serialization import from_dict, to_dict +from answerable.evidence.claims import ClaimClass, ClaimContext +from answerable.evidence.verdict import FindingInput, Repairability, VerdictEngine +from tests.e2e.test_runner_edge_cases import _case + + +def test_INV_002_integrity_blocker_dominates_even_innocent_wording() -> None: + blocker = FindingInput( + "duplicate_entities", + "data_integrity", + "blocker", + "duplicates", + True, + Repairability.RECOVERABLE, + ) + claim = "Observed revenue was higher." + result = VerdictEngine().decide( + (blocker,), claims=((claim, ClaimContext(ClaimClass.DESCRIPTIVE, "customers", "2025")),) + ) + assert result.verdict is Verdict.DATA_INTEGRITY_FAILURE + assert result.allowed_claims == () + assert result.forbidden_claims == (claim,) + + +def test_INV_002_missing_evidence_blocks_causal_claim_without_trigger_words() -> None: + blocker = FindingInput( + "immature_cohort", + "missing_evidence", + "blocker", + "immature", + False, + Repairability.RECOVERABLE, + ) + claim = "Exposure improved retention." + result = VerdictEngine().decide( + (blocker,), + claims=((claim, ClaimContext(ClaimClass.CAUSAL, "customers", "2025", causal_gate=True)),), + ) + assert result.allowed_claims == () + assert result.forbidden_claims == (claim,) + + +def test_INV_010_partial_overlap_does_not_identify_the_whole_population(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + mapping = load_spec(root / "question.json").mapping + rows = ( + {"treatment": "true", "acquisition_channel": "shared"}, + {"treatment": "false", "acquisition_channel": "shared"}, + {"treatment": "true", "acquisition_channel": "exclusive"}, + ) + assert not _overlap(rows, mapping) + + +def test_INV_008_overlap_is_not_exchangeability(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), + spec=load_spec(root / "question.json"), + output_directory=root / "out", + ) + assert run.verdict is Verdict.NOT_ANSWERABLE_YET + assert "identification_evidence_missing" in {b.finding_id for b in run.blockers} + assert run.allowed_claims == () + + +def test_INV_009_descriptive_question_does_not_require_causal_identification( + tmp_path: Path, +) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=8) + spec = load_spec(root / "question.json") + spec = replace( + spec, + contract=replace(spec.contract, analysis_type=AnalysisType.DESCRIPTIVE), + claims=(ClaimCandidate("Observed group means differ.", ClaimClass.DESCRIPTIVE),), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert run.verdict is Verdict.ANSWERABLE + assert run.allowed_claims == ("Observed group means differ.",) + + +def test_INV_001_findings_point_to_the_check_that_computed_them(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-06-29T00:00:00+00:00", size=8) + AssessmentRunner().run( + data_sources=(root / "customers.csv",), + spec=load_spec(root / "question.json"), + output_directory=root / "out", + ) + graph = json.loads((root / "out/evidence_graph.json").read_text()) + edges = {(e["source"], e["target"]) for e in graph["edges"] if e["type"] == "depends_on"} + assert ("finding_immature_cohort", "chk_temporal_maturity") in edges + assert ("finding_insufficient_power", "chk_statistical_power") in edges + assert ("finding_immature_cohort", "chk_positivity_overlap") not in edges + + +def test_INV_008_assumptions_never_become_facts_and_change_identity(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + spec = load_spec(root / "question.json") + old = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "old" + ) + spec = replace( + spec, + causal=replace(spec.causal, identification_assumptions=("conditional_exchangeability",)), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "new" + ) + assert old.assessment_id != run.assessment_id + assert run.verdict is Verdict.ANSWERABLE_WITH_ASSUMPTIONS + assert run.allowed_claims + ledger = run.observations["evidence_status"] + assert "conditional_exchangeability" in ledger["declared_assumptions"] + assert ledger["unverifiable_conditions"] + assert all("exchangeability" not in f["id"] for f in ledger["verified_facts"]) + graph = json.loads((root / "new/evidence_graph.json").read_text()) + assert {"fact", "assumption", "unverifiable_condition"} <= {n["type"] for n in graph["nodes"]} + checks = { + c["check_id"] for c in json.loads((root / "new/check_plan.json").read_text())["checks"] + } + assert checks == {n["id"] for n in graph["nodes"] if n["type"] == "execution"} + assert from_dict(CausalContract, to_dict(spec.causal)) == spec.causal + + +def test_INV_010_assuming_exchangeability_cannot_override_missing_overlap(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + spec = load_spec(root / "question.json") + spec = replace( + spec, + mapping=replace(spec.mapping, covariate_columns=("customer_id",)), + causal=replace( + spec.causal, + adjustment_set=frozenset({"customer_id"}), + identification_assumptions=("conditional_exchangeability",), + ), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert run.verdict is Verdict.FUNDAMENTALLY_UNIDENTIFIABLE + assert not run.allowed_claims + + +def test_INV_010_unchecked_adjustment_columns_block_identification(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + spec = load_spec(root / "question.json") + spec = replace( + spec, + causal=replace( + spec.causal, + adjustment_set=frozenset({"unobserved"}), + identification_assumptions=("conditional_exchangeability",), + ), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert "adjustment_mapping_mismatch" in {b.finding_id for b in run.blockers} + assert not run.allowed_claims + + +@pytest.mark.parametrize("levels", [(), ("a",), ("a", "b", "c"), ("a", None)]) +def test_INV_010_overlap_requires_two_non_null_levels(tmp_path: Path, levels: tuple) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + mapping = replace(load_spec(root / "question.json").mapping, covariate_columns=()) + assert not _overlap(tuple({"treatment": level} for level in levels), mapping) + + +def test_INV_008_legacy_contract_and_invalid_assumption_schema(tmp_path: Path) -> None: + import jsonschema + + root = _case(tmp_path, "2025-01-10T00:00:00+00:00") + contract = load_spec(root / "question.json").causal + assert contract.identification_assumptions == () + schema = json.loads( + (Path(__file__).parents[1] / "schemas/v1/causal-contract.schema.json").read_text() + ) + payload = to_dict(contract) + jsonschema.validate(payload, schema) + payload.pop("identification_assumptions") + jsonschema.validate(payload, schema) + assert from_dict(CausalContract, payload) == contract + with pytest.raises(ValueError, match="unknown identification assumption"): + replace(contract, identification_assumptions=("overlap_proves_causality",)) + with pytest.raises(TypeError, match="frozenset"): + from_dict(CausalContract, {**payload, "adjustment_set": "bad"}) + + +def test_INV_008_randomization_declaration_is_also_conditional(tmp_path: Path) -> None: + root = _case(tmp_path, "2025-01-10T00:00:00+00:00", size=160) + spec = load_spec(root / "question.json") + spec = replace( + spec, + causal=replace( + spec.causal, + strategy=IdentificationStrategy.RANDOMIZED, + identification_assumptions=("random_assignment",), + ), + ) + run = AssessmentRunner().run( + data_sources=(root / "customers.csv",), spec=spec, output_directory=root / "out" + ) + assert run.verdict is Verdict.ANSWERABLE_WITH_ASSUMPTIONS