From 94fc251abcc7a262534645294825dcc1f6d14720 Mon Sep 17 00:00:00 2001 From: Pranav Bhave <90584946+Cubits11@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:12:49 -0400 Subject: [PATCH] fix(e2): reject malformed observations with structured violations --- .../V1V2/CONTRACT.release-remainder.log | 9 + .../e2-wave1/V1V2/CONTRACT.test-release.log | 48 ++ docs/research/e2-wave1/V1V2/README.md | 48 ++ docs/research/e2-wave1/V1V2/audit.log | 28 + docs/research/e2-wave1/V1V2/nonfinite.pre.log | 34 ++ docs/research/e2-wave1/V1V2/post.log | 6 + docs/research/e2-wave1/V1V2/pre.log | 517 ++++++++++++++++++ scripts/validate_e2_observations.py | 79 ++- .../research/test_e2_validator_malformed.py | 164 ++++++ 9 files changed, 911 insertions(+), 22 deletions(-) create mode 100644 docs/research/e2-wave1/V1V2/CONTRACT.release-remainder.log create mode 100644 docs/research/e2-wave1/V1V2/CONTRACT.test-release.log create mode 100644 docs/research/e2-wave1/V1V2/README.md create mode 100644 docs/research/e2-wave1/V1V2/audit.log create mode 100644 docs/research/e2-wave1/V1V2/nonfinite.pre.log create mode 100644 docs/research/e2-wave1/V1V2/post.log create mode 100644 docs/research/e2-wave1/V1V2/pre.log create mode 100644 tests/unit/research/test_e2_validator_malformed.py diff --git a/docs/research/e2-wave1/V1V2/CONTRACT.release-remainder.log b/docs/research/e2-wave1/V1V2/CONTRACT.release-remainder.log new file mode 100644 index 0000000..d378b73 --- /dev/null +++ b/docs/research/e2-wave1/V1V2/CONTRACT.release-remainder.log @@ -0,0 +1,9 @@ +DIAGNOSTIC ONLY: test-kernel prerequisite skipped after its separate required-gate failure; this does not make test-release pass. +COMMAND: PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTHONDONTWRITEBYTECODE=1 make -o install -o test-kernel VENV_DIR=/tmp/cc-validation-20261023 test-release +PYTHONPATH=src /tmp/cc-validation-20261023/bin/python examples/minimal/run_bounds.py >/dev/null +PYTHONPATH=src /tmp/cc-validation-20261023/bin/pytest \ + tests/integration/test_reproduce_paper.py \ + tests/integration/test_verify_paper_artifacts.py -q +.......... [100%] + +EXIT_CODE=0 diff --git a/docs/research/e2-wave1/V1V2/CONTRACT.test-release.log b/docs/research/e2-wave1/V1V2/CONTRACT.test-release.log new file mode 100644 index 0000000..9e7b995 --- /dev/null +++ b/docs/research/e2-wave1/V1V2/CONTRACT.test-release.log @@ -0,0 +1,48 @@ +COMMAND: PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTHONDONTWRITEBYTECODE=1 make -o install VENV_DIR=/tmp/cc-validation-20261023 test-release +PYTHONPATH=src /tmp/cc-validation-20261023/bin/pytest tests/unit/kernel -q +........................................................................ [ 47%] +........................................................................ [ 95%] +....... [100%] +PYTHONPATH=src /tmp/cc-validation-20261023/bin/pytest tests/unit/kernel/test_sensitivity.py tests/unit/kernel/test_frechet_classes.py -q +.............................................................. [100%] +PYTHONPATH=src /tmp/cc-validation-20261023/bin/mypy src/cc/kernel --strict +src/cc/kernel/frechet_classes.py:441: error: Redundant cast to +"ndarray[tuple[Any, ...], dtype[float64]]" [redundant-cast] + return cast(FloatArray, clipped) + ^ +src/cc/kernel/frechet_classes.py:586: error: Returning Any from function +declared to return "ndarray[tuple[Any, ...], dtype[float64]]" [no-any-return] + return np.all(states == 1.0, axis=1).astype(float) + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +src/cc/kernel/frechet_classes.py:587: error: Returning Any from function +declared to return "ndarray[tuple[Any, ...], dtype[float64]]" [no-any-return] + return np.any(states == 1.0, axis=1).astype(float) + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +src/cc/kernel/frechet_classes.py:631: error: Redundant cast to +"ndarray[tuple[Any, ...], dtype[float64]]" [redundant-cast] + return cast(FloatArray, normalized) + ^ +src/cc/kernel/cliff.py:596: error: Returning Any from function declared to +return "ndarray[tuple[Any, ...], dtype[float64]]" [no-any-return] + return log_c + ^~~~~~~~~~~~ +src/cc/kernel/cliff.py:703: error: Returning Any from function declared to +return "ndarray[tuple[Any, ...], dtype[float64]]" [no-any-return] + return numerator / denominator * factor + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +src/cc/kernel/stress.py:611: error: Returning Any from function declared to +return "ndarray[tuple[Any, ...], dtype[float64]]" [no-any-return] + return np.mean(np.abs(states[:, None, :] - states[None, :, :]), ax... + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~... +src/cc/kernel/stress.py:706: error: Redundant cast to +"ndarray[tuple[Any, ...], dtype[float64]]" [redundant-cast] + return cast(FloatArray, pmf / float(np.sum(pmf))) + ^ +src/cc/kernel/ccf_models.py:347: error: Redundant cast to +"ndarray[tuple[Any, ...], dtype[Any]]" [redundant-cast] + return cast(np.ndarray, np.clip(arr, 0.0, 1.0).astype(float, copy=... + ^ +Found 9 errors in 4 files (checked 12 source files) +make: *** [test-kernel] Error 1 + +EXIT_CODE=2 diff --git a/docs/research/e2-wave1/V1V2/README.md b/docs/research/e2-wave1/V1V2/README.md new file mode 100644 index 0000000..8f71654 --- /dev/null +++ b/docs/research/e2-wave1/V1V2/README.md @@ -0,0 +1,48 @@ +# V1/V2: structured validator failures and checked timestamps + +Base: CONTRACT E1 `d060fee`; applied after SITE E5 `1fe36e6`. + +`pre.log`: **exit 1, 24 failed / 19 passed**. The corpus reproduces malformed +replicate/outcome tracebacks, unstructured malformed-file errors, and invalid +timestamps accepted with exit 0. `nonfinite.pre.log`: **exit 1, 3 failed**; +non-JSON NaN/Infinity tokens nested in raw detail were accepted before strict +JSON parsing. + +`scripts/validate_e2_observations.py` now reports input/schema violations before +running design checks or audit calculations. Malformed rows never reach the +replicate hash/sort or outcome integer coercion. Input read/decode errors and +empty inputs produce structured violations with exit 1. JSON mode writes one +JSON document to stdout. Valid datasets retain the same audit and warnings +(`audit.log`, including strict JSON decoding of both reference runs). + +The schema validator receives a `FormatChecker`. If jsonschema's optional +date-time checker is absent, a registered stdlib fallback checks the timestamp +shape, calendar date, time, and explicit UTC/numeric offset. No validator +dependency was added. Syntax reference: +[RFC 3339 sections 5.6–5.7](https://www.rfc-editor.org/rfc/rfc3339#section-5.6). + +`post.log`: **exit 0, 82 passed**, with no skips. This includes the malformed +corpus, E1 interoperability, existing observation-contract tests, and dry-run +tests. Ruff and whitespace checks pass (`audit.log`). No scientific constants, +thresholds, outcomes, sample sizes, or frozen preregistration text changed. + +```sh +E2_SITE_ROOT=/Users/pranavbhave/Developer/GitHub/e2-wave1-site \ +PYTHONPATH=/tmp/e2-wave1-20261024/test-dependencies:src:. \ +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 \ +/tmp/cc-validation-20261023/bin/python -B -m pytest \ + -o addopts= -p no:cacheprovider -q -rs \ + tests/unit/research/test_e2_validator_malformed.py \ + tests/unit/research/test_e2_stratum.py \ + tests/unit/research/test_e2_observation_contract.py \ + tests/unit/research/test_e2_dryrun.py +``` + +Repository release checks are recorded separately from the frozen E2 GO +criteria. `CONTRACT.test-release.log` records the required command **exit 2**: +kernel unit tests pass; strict mypy reports nine errors in four unchanged +kernel files. Kernel source equality to base `bf5740c` is checked in `audit.log`. +No unrelated fixes were made. `CONTRACT.release-remainder.log` records a +diagnostic run with that failed prerequisite explicitly skipped: the minimal +example and ten release integration tests pass, **exit 0**. The required +combined release check remains failed. diff --git a/docs/research/e2-wave1/V1V2/audit.log b/docs/research/e2-wave1/V1V2/audit.log new file mode 100644 index 0000000..054e80b --- /dev/null +++ b/docs/research/e2-wave1/V1V2/audit.log @@ -0,0 +1,28 @@ +PASS: date-time registered in FormatChecker=True +COMMAND: /tmp/cc-validation-20261023/bin/python -B scripts/validate_e2_observations.py examples/e2/observations.example.jsonl --json +PASS: clean JSON stdout; exit=0; zero violations; audit and warnings exactly equal pre-V1V2 HEAD; rows=24 +COMMAND: /tmp/cc-validation-20261023/bin/python -B scripts/validate_e2_observations.py experiments/e2_dryrun/results/observations.jsonl --json +PASS: clean JSON stdout; exit=0; zero violations; audit and warnings exactly equal pre-V1V2 HEAD; rows=66 +COMMAND: /tmp/cc-validation-20261023/bin/python -B scripts/validate_e2_observations.py /tmp/e2-wave1-20261024/does-not-exist.jsonl --json +{ + "report": null, + "violations": [ + "input violation: FileNotFoundError: [Errno 2] No such file or directory: '/tmp/e2-wave1-20261024/does-not-exist.jsonl'" + ], + "warnings": [] +} + +STDERR: +E2 conformance FAILED with 1 violation(s) + +EXIT_CODE=1; PASS structured rejection; audit not computed +COMMAND: /tmp/cc-validation-20261023/bin/python -m ruff check scripts/validate_e2_observations.py tests/unit/research/test_e2_validator_malformed.py +All checks passed! + +EXIT_CODE=0 +COMMAND: git diff --check + +EXIT_CODE=0 +COMMAND: git diff --exit-code bf5740c -- src/cc/kernel + +EXIT_CODE=0 diff --git a/docs/research/e2-wave1/V1V2/nonfinite.pre.log b/docs/research/e2-wave1/V1V2/nonfinite.pre.log new file mode 100644 index 0000000..5092c3f --- /dev/null +++ b/docs/research/e2-wave1/V1V2/nonfinite.pre.log @@ -0,0 +1,34 @@ +COMMAND: PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /tmp/cc-validation-20261023/bin/python -B -m pytest -o addopts= -p no:cacheprovider -q --tb=short tests/unit/research/test_e2_validator_malformed.py -k nonstandard_json_number +FFF [100%] +=================================== FAILURES =================================== +_________ test_nonstandard_json_number_is_a_structured_violation[NaN] __________ +tests/unit/research/test_e2_validator_malformed.py:128: in test_nonstandard_json_number_is_a_structured_violation + _reject(_run(_write_rows(tmp_path, rows))) +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c..., gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."\n ]\n}\n', stderr='').returncode +_______ test_nonstandard_json_number_is_a_structured_violation[Infinity] _______ +tests/unit/research/test_e2_validator_malformed.py:128: in test_nonstandard_json_number_is_a_structured_violation + _reject(_run(_write_rows(tmp_path, rows))) +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c..., gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."\n ]\n}\n', stderr='').returncode +______ test_nonstandard_json_number_is_a_structured_violation[-Infinity] _______ +tests/unit/research/test_e2_validator_malformed.py:128: in test_nonstandard_json_number_is_a_structured_violation + _reject(_run(_write_rows(tmp_path, rows))) +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c..., gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."\n ]\n}\n', stderr='').returncode +=========================== short test summary info ============================ +FAILED tests/unit/research/test_e2_validator_malformed.py::test_nonstandard_json_number_is_a_structured_violation[NaN] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_nonstandard_json_number_is_a_structured_violation[Infinity] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_nonstandard_json_number_is_a_structured_violation[-Infinity] +3 failed, 43 deselected in 0.46s + +EXIT_CODE=1 diff --git a/docs/research/e2-wave1/V1V2/post.log b/docs/research/e2-wave1/V1V2/post.log new file mode 100644 index 0000000..389b030 --- /dev/null +++ b/docs/research/e2-wave1/V1V2/post.log @@ -0,0 +1,6 @@ +COMMAND: E2_SITE_ROOT=/Users/pranavbhave/Developer/GitHub/e2-wave1-site PYTHONPATH=/tmp/e2-wave1-20261024/test-dependencies:src:. PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /tmp/cc-validation-20261023/bin/python -B -m pytest -o addopts= -p no:cacheprovider -q -rs tests/unit/research/test_e2_validator_malformed.py tests/unit/research/test_e2_stratum.py tests/unit/research/test_e2_observation_contract.py tests/unit/research/test_e2_dryrun.py +........................................................................ [ 87%] +.......... [100%] +82 passed in 7.97s + +EXIT_CODE=0 diff --git a/docs/research/e2-wave1/V1V2/pre.log b/docs/research/e2-wave1/V1V2/pre.log new file mode 100644 index 0000000..7811bc8 --- /dev/null +++ b/docs/research/e2-wave1/V1V2/pre.log @@ -0,0 +1,517 @@ +COMMAND: PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /tmp/cc-validation-20261023/bin/python -B -m pytest -o addopts= -p no:cacheprovider -q --tb=short tests/unit/research/test_e2_validator_malformed.py +FF....FFFF...........FFFFFFFFFFFFFFFFFF.... [100%] +=================================== FAILURES =================================== +_________ test_malformed_replicate_is_structured_violation[replicate0] _________ +tests/unit/research/test_e2_validator_malformed.py:63: in test_malformed_replicate_is_structured_violation + _reject(_run(_write_rows(tmp_path, rows)), "replicate_id") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 240, in main +E violations = schema_violations(rows, schema) + design_violations(rows) +E ^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 107, in design_violations +E keys = Counter( +E ^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/collections/__init__.py", line 607, in __init__ +E self.update(iterable, **kwds) +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/collections/__init__.py", line 699, in update +E _count_elements(self, iterable) +E TypeError: unhashable type: 'list' +E +E assert 'Traceback' not in "Traceback (...pe: 'list'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (11 lines hidden), use '-vv' to show +_________ test_malformed_replicate_is_structured_violation[replicate1] _________ +tests/unit/research/test_e2_validator_malformed.py:63: in test_malformed_replicate_is_structured_violation + _reject(_run(_write_rows(tmp_path, rows)), "replicate_id") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 240, in main +E violations = schema_violations(rows, schema) + design_violations(rows) +E ^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 107, in design_violations +E keys = Counter( +E ^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/collections/__init__.py", line 607, in __init__ +E self.update(iterable, **kwds) +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/collections/__init__.py", line 699, in update +E _count_elements(self, iterable) +E TypeError: unhashable type: 'dict' +E +E assert 'Traceback' not in "Traceback (...pe: 'dict'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (11 lines hidden), use '-vv' to show +_______________ test_mixed_replicate_types_do_not_crash_sorting ________________ +tests/unit/research/test_e2_validator_malformed.py:69: in test_mixed_replicate_types_do_not_crash_sorting + _reject(_run(_write_rows(tmp_path, rows)), "replicate_id") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 240, in main +E violations = schema_violations(rows, schema) + design_violations(rows) +E ^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 111, in design_violations +E for key, count in sorted(keys.items()): +E ^^^^^^^^^^^^^^^^^^^^ +E TypeError: '<' not supported between instances of 'str' and 'int' +E +E assert 'Traceback' not in "Traceback (... and 'int'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (7 lines hidden), use '-vv' to show +_____________ test_malformed_outcome_is_structured_violation[junk] _____________ +tests/unit/research/test_e2_validator_malformed.py:76: in test_malformed_outcome_is_structured_violation + _reject(_run(_write_rows(tmp_path, rows)), "normalized_outcome") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 241, in main +E report = audit(rows) +E ^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 154, in audit +E replicate_outcomes[key].add(int(row["normalized_outcome"])) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E ValueError: invalid literal for int() with base 10: 'junk' +E +E assert 'Traceback' not in "Traceback (...10: 'junk'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (7 lines hidden), use '-vv' to show +___________ test_malformed_outcome_is_structured_violation[outcome1] ___________ +tests/unit/research/test_e2_validator_malformed.py:76: in test_malformed_outcome_is_structured_violation + _reject(_run(_write_rows(tmp_path, rows)), "normalized_outcome") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 241, in main +E report = audit(rows) +E ^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 154, in audit +E replicate_outcomes[key].add(int(row["normalized_outcome"])) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E TypeError: int() argument must be a string, a bytes-like object or a real number, not 'list' +E +E assert 'Traceback' not in "Traceback (...not 'list'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (7 lines hidden), use '-vv' to show +___________ test_malformed_outcome_is_structured_violation[outcome2] ___________ +tests/unit/research/test_e2_validator_malformed.py:76: in test_malformed_outcome_is_structured_violation + _reject(_run(_write_rows(tmp_path, rows)), "normalized_outcome") +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 241, in main +E report = audit(rows) +E ^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 154, in audit +E replicate_outcomes[key].add(int(row["normalized_outcome"])) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E TypeError: int() argument must be a string, a bytes-like object or a real number, not 'dict' +E +E assert 'Traceback' not in "Traceback (...not 'dict'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (7 lines hidden), use '-vv' to show +__________ test_non_object_jsonl_rows_are_structured_violations[row0] __________ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:104: in test_non_object_jsonl_rows_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_non_object_jsonl_rows_are0/not-an-object.jsonl:1: each line must be a JSON object\n' +__________ test_non_object_jsonl_rows_are_structured_violations[None] __________ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:104: in test_non_object_jsonl_rows_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_non_object_jsonl_rows_are1/not-an-object.jsonl:1: each line must be a JSON object\n' +__________ test_non_object_jsonl_rows_are_structured_violations[row] ___________ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:104: in test_non_object_jsonl_rows_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_non_object_jsonl_rows_are2/not-an-object.jsonl:1: each line must be a JSON object\n' +___________ test_non_object_jsonl_rows_are_structured_violations[1] ____________ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:104: in test_non_object_jsonl_rows_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_non_object_jsonl_rows_are3/not-an-object.jsonl:1: each line must be a JSON object\n' +__________ test_non_object_jsonl_rows_are_structured_violations[True] __________ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:104: in test_non_object_jsonl_rows_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_non_object_jsonl_rows_are4/not-an-object.jsonl:1: each line must be a JSON object\n' +_ test_unreadable_or_empty_observations_are_structured_violations[{"item_id":] _ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:113: in test_unreadable_or_empty_observations_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_unreadable_or_empty_obser0/malformed.jsonl:1: invalid JSON: Expecting value: line 1 column 12 (char 11)\n' +_ test_unreadable_or_empty_observations_are_structured_violations[\xff\xfe\n] __ +tests/unit/research/test_e2_validator_malformed.py:113: in test_unreadable_or_empty_observations_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 235, in main +E rows = load_rows(args.observations) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 40, in load_rows +E for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/pathlib.py", line 1028, in read_text +E return f.read() +E ^^^^^^^^ +E File "", line 322, in decode +E UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte +E +E assert 'Traceback' not in 'Traceback (...start byte\n' +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (11 lines hidden), use '-vv' to show +______ test_unreadable_or_empty_observations_are_structured_violations[] _______ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:113: in test_unreadable_or_empty_observations_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='E2 conformance error: no observation rows\n' +__ test_unreadable_or_empty_observations_are_structured_violations[ \n\t\n] ___ +tests/unit/research/test_e2_validator_malformed.py:36: in _payload + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +../../../.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/json/decoder.py:356: in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +During handling of the above exception, another exception occurred: +tests/unit/research/test_e2_validator_malformed.py:113: in test_unreadable_or_empty_observations_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:38: in _payload + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") +E Failed: no structured validator output: stdout='', stderr='E2 conformance error: no observation rows\n' +___________ test_input_io_errors_are_structured_violations[missing] ____________ +tests/unit/research/test_e2_validator_malformed.py:121: in test_input_io_errors_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 235, in main +E rows = load_rows(args.observations) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 40, in load_rows +E for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/pathlib.py", line 1027, in read_text +E with self.open(mode='r', encoding=encoding, errors=errors) as f: +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/pathlib.py", line 1013, in open +E return io.open(self, mode, buffering, encoding, errors, newline) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E FileNotFoundError: [Errno 2] No such file or directory: '/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_input_io_errors_are_struc0/input.jsonl' +E +E assert 'Traceback' not in "Traceback (...put.jsonl'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (13 lines hidden), use '-vv' to show +__________ test_input_io_errors_are_structured_violations[directory] ___________ +tests/unit/research/test_e2_validator_malformed.py:121: in test_input_io_errors_are_structured_violations + _reject(_run(path)) +tests/unit/research/test_e2_validator_malformed.py:45: in _reject + payload = _payload(result) + ^^^^^^^^^^^^^^^^ +tests/unit/research/test_e2_validator_malformed.py:32: in _payload + assert "Traceback" not in result.stdout + result.stderr, result.stderr +E AssertionError: Traceback (most recent call last): +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 235, in main +E rows = load_rows(args.observations) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 40, in load_rows +E for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/pathlib.py", line 1027, in read_text +E with self.open(mode='r', encoding=encoding, errors=errors) as f: +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E File "/Users/pranavbhave/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/lib/python3.12/pathlib.py", line 1013, in open +E return io.open(self, mode, buffering, encoding, errors, newline) +E ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E IsADirectoryError: [Errno 21] Is a directory: '/private/var/folders/nj/2c1_c61j3hn7h5fgv1gk0q2m0000gn/T/pytest-of-pranavbhave/pytest-16/test_input_io_errors_are_struc1/input.jsonl' +E +E assert 'Traceback' not in "Traceback (...put.jsonl'\n" +E +E 'Traceback' is contained here: +E Traceback (most recent call last): +E ? +++++++++ +E File "/Users/pranavbhave/Developer/GitHub/e2-wave1-contract/scripts/validate_e2_observations.py", line 265, in +E raise SystemExit(main()) +E ^^^^^^... +E +E ...Full output truncated (13 lines hidden), use '-vv' to show +__________ test_invalid_rfc3339_timestamp_is_a_violation[not-a-date] ___________ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +__________ test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24] ___________ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +______ test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00] ______ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +_____ test_invalid_rfc3339_timestamp_is_a_violation[2026-13-24T20:00:00Z] ______ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +_____ test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T25:00:00Z] ______ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +___ test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00+24:00] ___ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +____ test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00+04] _____ +tests/unit/research/test_e2_validator_malformed.py:139: in test_invalid_rfc3339_timestamp_is_a_violation + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") +tests/unit/research/test_e2_validator_malformed.py:46: in _reject + assert result.returncode == 1, (result.returncode, payload, result.stderr) +E AssertionError: (0, {'report': {'rows': 3, 'guardrails': ['gr.alpha', 'gr.beta', 'gr.gamma'], 'items': 1, 'complete_case_items': 1, .....' spans gr.alpha, gr.gamma; observed dependence may be mechanism-induced rather than a property of composition."]}, '') +E assert 0 == 1 +E + where 0 = CompletedProcess(args=['/tmp/cc-validation-20261023/bin/python', '-B', '/Users/pranavbhave/Developer/GitHub/e2-wave1-c...her than a property of composition."\n ]\n}\n\nE2 conformance OK (warnings are disclosures, not passes)\n', stderr='').returncode +=========================== short test summary info ============================ +FAILED tests/unit/research/test_e2_validator_malformed.py::test_malformed_replicate_is_structured_violation[replicate0] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_malformed_replicate_is_structured_violation[replicate1] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_mixed_replicate_types_do_not_crash_sorting +FAILED tests/unit/research/test_e2_validator_malformed.py::test_malformed_outcome_is_structured_violation[junk] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_malformed_outcome_is_structured_violation[outcome1] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_malformed_outcome_is_structured_violation[outcome2] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_non_object_jsonl_rows_are_structured_violations[row0] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_non_object_jsonl_rows_are_structured_violations[None] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_non_object_jsonl_rows_are_structured_violations[row] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_non_object_jsonl_rows_are_structured_violations[1] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_non_object_jsonl_rows_are_structured_violations[True] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_unreadable_or_empty_observations_are_structured_violations[{"item_id":] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_unreadable_or_empty_observations_are_structured_violations[\xff\xfe\n] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_unreadable_or_empty_observations_are_structured_violations[] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_unreadable_or_empty_observations_are_structured_violations[ \n\t\n] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_input_io_errors_are_structured_violations[missing] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_input_io_errors_are_structured_violations[directory] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[not-a-date] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-13-24T20:00:00Z] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T25:00:00Z] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00+24:00] +FAILED tests/unit/research/test_e2_validator_malformed.py::test_invalid_rfc3339_timestamp_is_a_violation[2026-10-24T20:00:00+04] +24 failed, 19 passed in 4.65s + +EXIT_CODE=1 diff --git a/scripts/validate_e2_observations.py b/scripts/validate_e2_observations.py index fd0e6cb..2964b4e 100755 --- a/scripts/validate_e2_observations.py +++ b/scripts/validate_e2_observations.py @@ -18,14 +18,34 @@ import argparse import json +import re import sys from collections import Counter, defaultdict +from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, NoReturn import jsonschema DEFAULT_SCHEMA = Path("schemas/cc.e2_observation_row.v1.json") +FORMAT_CHECKER = jsonschema.FormatChecker() + +# jsonschema's date-time checker depends on an optional package. Register a +# stdlib fallback when it is absent, rather than silently ignoring the format. +if "date-time" not in FORMAT_CHECKER.checkers: + _RFC3339 = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}[Tt](?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" + r"(?:\.[0-9]+)?(?:[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])" + ) + + @FORMAT_CHECKER.checks("date-time", raises=ValueError) + def _is_rfc3339(value: Any) -> bool: + if not isinstance(value, str): + return True # The schema's type check reports non-string values. + if _RFC3339.fullmatch(value) is None: + return False + datetime.fromisoformat(value.upper().replace("Z", "+00:00")) + return True # Missingness codes whose presence changes how a dependence estimate must be read. INFORMATIVE_MISSINGNESS = frozenset( @@ -33,6 +53,10 @@ ) +def _reject_json_constant(value: str) -> NoReturn: + raise ValueError(f"non-JSON numeric constant {value!r}") + + def load_rows(path: Path) -> list[dict[str, Any]]: """Read a JSONL observation file into a list of row mappings.""" @@ -42,11 +66,11 @@ def load_rows(path: Path) -> list[dict[str, Any]]: if not stripped: continue try: - parsed = json.loads(stripped) - except json.JSONDecodeError as exc: - raise SystemExit(f"{path}:{number}: invalid JSON: {exc}") from exc + parsed = json.loads(stripped, parse_constant=_reject_json_constant) + except ValueError as exc: + raise ValueError(f"{path}:{number}: invalid JSON: {exc}") from exc if not isinstance(parsed, dict): - raise SystemExit(f"{path}:{number}: each line must be a JSON object") + raise ValueError(f"{path}:{number}: each line must be a JSON object") rows.append(parsed) return rows @@ -54,10 +78,10 @@ def load_rows(path: Path) -> list[dict[str, Any]]: def schema_violations(rows: list[dict[str, Any]], schema: dict[str, Any]) -> list[str]: """Return schema violations, including the required typed evaluation stratum.""" - validator = jsonschema.Draft202012Validator(schema) + validator = jsonschema.Draft202012Validator(schema, format_checker=FORMAT_CHECKER) problems: list[str] = [] for index, row in enumerate(rows, start=1): - for error in sorted(validator.iter_errors(row), key=lambda err: list(err.path)): + for error in sorted(validator.iter_errors(row), key=lambda err: tuple(map(str, err.path))): location = "/".join(str(part) for part in error.path) or "" problems.append(f"row {index}: {location}: {error.message}") return problems @@ -231,24 +255,34 @@ def main() -> int: parser.add_argument("--json", action="store_true", help="Emit the audit report as JSON") args = parser.parse_args() - schema = json.loads(args.schema.read_text(encoding="utf-8")) - rows = load_rows(args.observations) - if not rows: - print("E2 conformance error: no observation rows", file=sys.stderr) - return 1 - - violations = schema_violations(rows, schema) + design_violations(rows) - report = audit(rows) - notes = warnings_from(report) + report = None + notes: list[str] = [] + try: + schema = json.loads( + args.schema.read_text(encoding="utf-8"), parse_constant=_reject_json_constant + ) + jsonschema.Draft202012Validator.check_schema(schema) + rows = load_rows(args.observations) + except (OSError, UnicodeError, ValueError, jsonschema.SchemaError) as exc: + violations = [f"input violation: {type(exc).__name__}: {exc}"] + else: + violations = schema_violations(rows, schema) if rows else ["no observation rows"] + # Design keys and audit calculations require schema-valid types. Never + # hash malformed replicate ids or coerce malformed outcomes into bits. + if not violations: + violations = design_violations(rows) + report = audit(rows) + notes = warnings_from(report) if args.json: print(json.dumps({"report": report, "violations": violations, "warnings": notes}, indent=2)) else: - print(f"rows={report['rows']} items={report['items']} guardrails={report['guardrails']}") - print( - f"complete-case items: {report['complete_case_items']}/{report['items']} " - f"({report['complete_case_fraction']:.1%})" - ) + if report is not None: + print(f"rows={report['rows']} items={report['items']} guardrails={report['guardrails']}") + print( + f"complete-case items: {report['complete_case_items']}/{report['items']} " + f"({report['complete_case_fraction']:.1%})" + ) for note in notes: print(f" warning: {note}") for violation in violations: @@ -257,7 +291,8 @@ def main() -> int: if violations: print(f"\nE2 conformance FAILED with {len(violations)} violation(s)", file=sys.stderr) return 1 - print("\nE2 conformance OK (warnings are disclosures, not passes)") + if not args.json: + print("\nE2 conformance OK (warnings are disclosures, not passes)") return 0 diff --git a/tests/unit/research/test_e2_validator_malformed.py b/tests/unit/research/test_e2_validator_malformed.py new file mode 100644 index 0000000..008db79 --- /dev/null +++ b/tests/unit/research/test_e2_validator_malformed.py @@ -0,0 +1,164 @@ +"""V1/V2: malformed observations yield violations, and timestamps are RFC 3339.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +VALIDATOR = ROOT / "scripts/validate_e2_observations.py" +SCHEMA = ROOT / "schemas/cc.e2_observation_row.v1.json" +REFERENCE = ROOT / "examples/e2/observations.example.jsonl" + + +def _rows() -> list[dict]: + return [json.loads(line) for line in REFERENCE.read_text().splitlines()[:3]] + + +def _run(path: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-B", str(VALIDATOR), str(path), "--schema", str(SCHEMA), "--json"], + text=True, + capture_output=True, + check=False, + ) + + +def _payload(result: subprocess.CompletedProcess[str]) -> dict: + assert "Traceback" not in result.stdout + result.stderr, result.stderr + # Accept the existing success-summary suffix; the violations themselves + # must be a structured JSON document on stdout, never exception prose. + try: + payload, _ = json.JSONDecoder().raw_decode(result.stdout.lstrip()) + except json.JSONDecodeError: + pytest.fail(f"no structured validator output: stdout={result.stdout!r}, stderr={result.stderr!r}") + assert isinstance(payload, dict) + assert isinstance(payload.get("violations"), list) + return payload + + +def _reject(result: subprocess.CompletedProcess[str], field: str | None = None) -> None: + payload = _payload(result) + assert result.returncode == 1, (result.returncode, payload, result.stderr) + assert payload["violations"], payload + assert all(isinstance(message, str) for message in payload["violations"]) + if field is not None: + assert any(field in message for message in payload["violations"]), payload + + +def _write_rows(tmp_path: Path, rows: list[dict]) -> Path: + path = tmp_path / "observations.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + return path + + +@pytest.mark.parametrize("replicate", [[], {}, "not-an-int", None, 0.5, True]) +def test_malformed_replicate_is_structured_violation(tmp_path: Path, replicate: object) -> None: + rows = _rows() + rows[0]["replicate_id"] = replicate + _reject(_run(_write_rows(tmp_path, rows)), "replicate_id") + + +def test_mixed_replicate_types_do_not_crash_sorting(tmp_path: Path) -> None: + rows = _rows() + rows.append(dict(rows[0], replicate_id="1")) + _reject(_run(_write_rows(tmp_path, rows)), "replicate_id") + + +@pytest.mark.parametrize("outcome", ["junk", [], {}, True, 1.5, None]) +def test_malformed_outcome_is_structured_violation(tmp_path: Path, outcome: object) -> None: + rows = _rows() + rows[0]["normalized_outcome"] = outcome + _reject(_run(_write_rows(tmp_path, rows)), "normalized_outcome") + + +@pytest.mark.parametrize( + "field,value", + [ + ("item_id", []), + ("guardrail_id", {}), + ("configuration_hash", []), + ("missingness_code", []), + ("shared_dependency_group", {}), + ("guardrail_version", []), + ("raw_outcome_detail", "score"), + ("execution_timestamp", 1), + ], +) +def test_other_malformed_fields_are_structured_violations( + tmp_path: Path, field: str, value: object +) -> None: + rows = _rows() + rows[0][field] = value + _reject(_run(_write_rows(tmp_path, rows)), field) + + +@pytest.mark.parametrize("row", [[], None, "row", 1, True]) +def test_non_object_jsonl_rows_are_structured_violations(tmp_path: Path, row: object) -> None: + path = tmp_path / "not-an-object.jsonl" + path.write_text(json.dumps(row) + "\n") + _reject(_run(path)) + + +@pytest.mark.parametrize("content", [b'{"item_id":', b'\xff\xfe\n', b'', b' \n\t\n']) +def test_unreadable_or_empty_observations_are_structured_violations( + tmp_path: Path, content: bytes +) -> None: + path = tmp_path / "malformed.jsonl" + path.write_bytes(content) + _reject(_run(path)) + + +@pytest.mark.parametrize("kind", ["missing", "directory"]) +def test_input_io_errors_are_structured_violations(tmp_path: Path, kind: str) -> None: + path = tmp_path / "input.jsonl" + if kind == "directory": + path.mkdir() + _reject(_run(path)) + + +@pytest.mark.parametrize("constant", ["NaN", "Infinity", "-Infinity"]) +def test_nonstandard_json_number_is_a_structured_violation(tmp_path: Path, constant: str) -> None: + rows = _rows() + rows[0]["raw_outcome_detail"] = {"score": float(constant)} + _reject(_run(_write_rows(tmp_path, rows))) + + +@pytest.mark.parametrize( + "timestamp", + [ + "not-a-date", + "2026-10-24", + "2026-10-24T20:00:00", + "2026-13-24T20:00:00Z", + "2026-10-24T25:00:00Z", + "2026-10-24T20:00:00+24:00", + "2026-10-24T20:00:00+04", + ], +) +def test_invalid_rfc3339_timestamp_is_a_violation(tmp_path: Path, timestamp: str) -> None: + rows = _rows() + rows[0]["execution_timestamp"] = timestamp + _reject(_run(_write_rows(tmp_path, rows)), "execution_timestamp") + + +@pytest.mark.parametrize( + "timestamp", + [ + "2026-10-24T20:00:00Z", + "2026-10-24T20:00:00.123456Z", + "2026-10-24T20:00:00-04:00", + "2026-10-24T20:00:00+05:30", + ], +) +def test_valid_rfc3339_timestamp_is_accepted(tmp_path: Path, timestamp: str) -> None: + rows = _rows() + rows[0]["execution_timestamp"] = timestamp + result = _run(_write_rows(tmp_path, rows)) + payload = _payload(result) + assert result.returncode == 0, (payload, result.stderr) + assert payload["violations"] == []