diff --git a/tests/test_distiller_payload_types.py b/tests/test_distiller_payload_types.py new file mode 100644 index 0000000..f5ea9a5 --- /dev/null +++ b/tests/test_distiller_payload_types.py @@ -0,0 +1,120 @@ +"""Malformed proposer payloads: wrongly-typed fields are dropped, never crash the distiller.""" +import json + +import pytest + +import tokeneconomizer.microskills.distiller as dist +from tokeneconomizer.microskills import Distiller +from tokeneconomizer.microskills.distiller import parse_payload +from fixtures import write_claude_transcript + + +def _runner(payload): + return lambda prompt, model="agent": (payload, 1.0) + + +def _payload(**body): + return json.dumps(body) + + +MALFORMED = [ + '{"extractors": null, "facts": null}', + _payload(extractors=[{"name": ["a"], "pattern": "x"}], facts=[]), + _payload(extractors=[{"name": {"x": 1}, "pattern": "x"}], facts=[]), + '{"extractors": true, "facts": []}', + '{"extractors": [], "facts": 7}', + _payload(extractors=[{"name": "a", "pattern": ""}], facts=[{"id": "f", "detector": {}}]), + _payload(extractors=[], facts=[{"id": ["a"], "detector": {"scope": "ran_tests"}}]), + _payload(extractors=[], facts=[{"id": {"x": 1}, "detector": {"scope": "ran_tests"}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": ["bad"]}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": {"a": 1}}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "fired": 5}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "confirm": ["x"]}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests", "contradict": {"x": 1}}}]), + _payload(extractors=[], facts=[{"id": "f", "detector": {"scope": "ran_tests"}, "cite": 3}]), +] + + +@pytest.mark.parametrize("raw", MALFORMED) +def test_parse_payload_drops_wrongly_typed_fields(raw): + exts, facts = parse_payload(raw) + assert exts == [] + assert facts == [] + + +def test_parse_payload_keeps_valid_entries_next_to_malformed(): + raw = _payload( + extractors=[{"name": ["a"], "pattern": "x"}, {"name": "ok", "pattern": "x"}], + facts=[{"id": {"x": 1}, "detector": {"scope": "ran_tests"}}, + {"id": "good", "detector": {"scope": "ran_tests", "confirm": "passed"}}]) + exts, facts = parse_payload(raw) + assert [e["name"] for e in exts] == ["ok"] + assert [f["id"] for f in facts] == ["good"] + + +def test_ground_treats_non_list_cite_as_empty(tmp_path): + d = Distiller("acme/widget") + tr = str(tmp_path / "t.jsonl") + write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")]) + d.update(tr, iid="w-0", resolved=True) + for cite in (3, "w-0", {}, {"w-0": 1}, [["w-0"], {"x": 1}, 7]): + spec = {"id": "f", "detector": {"scope": "ran_tests", "confirm": "passed"}, "cite": cite} + assert d._ground(spec) == 0 + + +@pytest.mark.parametrize("raw", MALFORMED) +def test_update_survives_malformed_proposer_reply(tmp_path, raw): + d = Distiller("acme/widget", runner=_runner(raw)) + d.last_propose = -1 + saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1 + try: + for i in range(3): + tr = str(tmp_path / f"t{i}.jsonl") + write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")]) + d.update(tr, iid=f"w-{i}", resolved=True) + finally: + dist.PROPOSE_EVERY = saved + assert d.facts == {} + assert d.extractors == {} + + +def test_valid_fact_admitted_alongside_malformed_ones(tmp_path): + raw = _payload( + extractors=[{"name": ["bad"], "pattern": "x"}], + facts=[{"id": ["bad"], "detector": {"scope": "ran_tests"}}, + {"id": "use-pytest", "claim": "pytest is canonical", "injection": "run pytest", + "detector": {"scope": "ran_tests", "confirm": "passed", "fired": "ran_tests"}, + "cite": ["w-0", "w-1"]}]) + d = Distiller("acme/widget", runner=_runner(raw)) + d.last_propose = -1 + saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1 + try: + for i in range(2): + tr = str(tmp_path / f"t{i}.jsonl") + write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")]) + d.update(tr, iid=f"w-{i}", resolved=True) + finally: + dist.PROPOSE_EVERY = saved + assert "use-pytest" in d.facts + assert set(d.facts) == {"use-pytest"} + + +def test_empty_detectors_do_not_crowd_out_a_valid_fact(tmp_path): + """An empty detector never grounds; it must not take one of the MAX_CANDS slots.""" + raw = _payload( + extractors=[], + facts=[*({"id": f"empty-{i}", "detector": {}} for i in range(4)), + {"id": "use-pytest", "claim": "pytest is canonical", "injection": "run pytest", + "detector": {"scope": "ran_tests", "confirm": "passed", "fired": "ran_tests"}, + "cite": ["w-0", "w-1"]}]) + d = Distiller("acme/widget", runner=_runner(raw)) + d.last_propose = -1 + saved, dist.PROPOSE_EVERY = dist.PROPOSE_EVERY, 1 + try: + for i in range(2): + tr = str(tmp_path / f"t{i}.jsonl") + write_claude_transcript(tr, [("bash", {"command": "pytest"}, "1 passed")]) + d.update(tr, iid=f"w-{i}", resolved=True) + finally: + dist.PROPOSE_EVERY = saved + assert set(d.facts) == {"use-pytest"} diff --git a/tokeneconomizer/microskills/distiller.py b/tokeneconomizer/microskills/distiller.py index 3005dda..80c3724 100644 --- a/tokeneconomizer/microskills/distiller.py +++ b/tokeneconomizer/microskills/distiller.py @@ -270,7 +270,7 @@ def proposer_view(dg): def _rx(pat): - if not pat: + if not pat or not isinstance(pat, str): return None try: return re.compile(pat) @@ -535,13 +535,23 @@ def parse_payload(raw): obj = json.loads(m.group(0)) except Exception: return [], [] - exts = [e for e in obj.get("extractors", []) - if isinstance(e, dict) and e.get("name") and e.get("pattern")] - facts = [f for f in obj.get("facts", []) - if isinstance(f, dict) and f.get("id") and f.get("detector")] + exts, facts = obj.get("extractors"), obj.get("facts") + exts = [e for e in (exts if isinstance(exts, list) else []) + if isinstance(e, dict) and isinstance(e.get("name"), str) and e.get("name") + and isinstance(e.get("pattern"), str) and e.get("pattern")] + facts = [f for f in (facts if isinstance(facts, list) else []) + if isinstance(f, dict) and isinstance(f.get("id"), str) and f.get("id") + and isinstance(f.get("detector"), dict) and f.get("detector") + and _detector_fields_ok(f["detector"]) + and (f.get("cite") is None or isinstance(f.get("cite"), list))] return exts, facts +def _detector_fields_ok(det): + return all(det.get(k) is None or isinstance(det.get(k), str) + for k in ("scope", "fired", "confirm", "contradict")) + + def _reparse_cmds(dg): """Recover [(cmd,out)] from a digest's stashed blobs (for retro-featurize). The blobs no longer say which output came from which command, so the whole output rides on the first @@ -672,7 +682,10 @@ def _ground(self, spec): """#window cites whose confirm regex fires under the CURRENT features.""" by_iid = {d["iid"]: d for d in self.window} hits = 0 - for iid in (spec.get("cite") or []): + cites = spec.get("cite") + for iid in (cites if isinstance(cites, list) else []): + if not isinstance(iid, str): + continue dg = by_iid.get(iid) if dg is None: continue