11"""Judge calls, all under the engine's ledger (design §10).
22
33The engine never starts a judge subprocess directly; every judge call goes
4- through `run_judge`, which has two paths:
4+ through `run_judge`, which has three paths:
55
66* **API judges** through `LLM.create` — captured as `model_call` records by
77 the choke point and priced against the dollar envelope (P3 governor);
88* **CLI judges** (a subscription harness such as `cursor-agent`) through
99 `governed_subprocess` — one call reserved against the judge-call envelope,
1010 tool-less by construction (fresh empty workspace, `--mode ask`, any tool
1111 call in the stream fails the verdict), recorded as a `model_call` with
12- ``usd=0``.
13-
14- A `Governor` (P3) gates both; without one the calls are unmetered but still
15- recorded. `parse_verdict` extracts the first JSON object of a reply.
12+ ``usd=0``;
13+ * **zcode judges** (``cli:zcode:<model>``) through the production
14+ `ZCodeTransport.complete` one-shot — the same tool-less instrument the
15+ knowledge gate's `ModelGateway` uses (empty scratch cwd, every native
16+ tool removed, containment audit, served-model assertion; the reasoning
17+ level is the ambient `ZCODE_REASONING_LEVEL`). Reserved and settled
18+ against the judge-call envelope like any CLI judge; subscription billing,
19+ so ``usd=0`` and only the call count is governed.
20+
21+ A `Governor` (P3) gates all of them; without one the calls are unmetered but
22+ still recorded. `parse_verdict` extracts the first JSON object of a reply.
1623"""
1724
1825from __future__ import annotations
@@ -38,7 +45,7 @@ class JudgeError(RuntimeError):
3845class JudgeSpec :
3946 kind : str # "api" | "cli"
4047 model : str
41- provider : str = "" # cli: cursor | codex | claude ; api: the LLM's provider
48+ provider : str = "" # cli: cursor | codex | claude | zcode ; api: the LLM's provider
4249 max_tokens : int = 2000
4350
4451
@@ -153,10 +160,38 @@ def governed_subprocess(argv: list[str], *, governor: Any = None, timeout: int =
153160 governor .settle_judge_call (token )
154161
155162
163+ def _run_zcode (spec : JudgeSpec , * , system : str , prompt : str , role : str ,
164+ governor : Any ) -> tuple [str , dict , str , str ]:
165+ """One governed zcode judge call through the production transport: the
166+ same tool-less one-shot the kb gate's ``ModelGateway`` uses (empty
167+ scratch cwd, every native tool removed, containment audit, served-model
168+ assertion), so an experiment's gate judge IS the production instrument,
169+ not an API proxy of it. Subscription billing: ``usd`` stays 0 and only
170+ the call count draws on the judge-call envelope."""
171+ from ..config import Settings
172+ from ..providers .registry import transport_for_id
173+
174+ token = governor .reserve_judge_call () if governor is not None else None
175+ try :
176+ transport = transport_for_id (Settings (), "zcode" )
177+ reply = transport .complete (system = system ,
178+ messages = [{"role" : "user" , "content" : prompt }],
179+ model = spec .model , role = role )
180+ text = "" .join (b .text for b in reply .blocks if b .type == "text" )
181+ usage = dict (getattr (reply , "usage" , None ) or {})
182+ return text , usage , "" if text else "empty zcode judge reply" , \
183+ str (getattr (reply , "model" , "" ) or "" )
184+ except Exception as exc : # noqa: BLE001 — the verdict records the reason
185+ return "" , {}, f"zcode judge unavailable: { type (exc ).__name__ } : { exc } " , ""
186+ finally :
187+ if governor is not None :
188+ governor .settle_judge_call (token )
189+
190+
156191def run_judge (spec : JudgeSpec , * , system : str , prompt : str , llm : Any = None , governor : Any = None ,
157192 role : str = "judge" , runner : Callable [..., Any ] | None = None ) -> dict :
158193 """Run one judge call and return its parsed JSON verdict (raises
159- JudgeError on an unusable reply). Both paths are recorded as
194+ JudgeError on an unusable reply). All paths are recorded as
160195 ``model_call`` records under the current trace context."""
161196 if spec .kind == "api" :
162197 if llm is None :
@@ -167,25 +202,31 @@ def run_judge(spec: JudgeSpec, *, system: str, prompt: str, llm: Any = None, gov
167202 return parse_verdict (text )
168203 if spec .kind != "cli" :
169204 raise JudgeError (f"unknown judge kind { spec .kind !r} " )
170- workspace = Path (tempfile .mkdtemp (prefix = "judge-ws-" ))
171- workspace .chmod (0o700 )
172205 started = time .monotonic ()
173- try :
174- full = prompt if not system else f"{ system } \n \n { prompt } "
175- proc = governed_subprocess (_cli_argv (spec , full , workspace ), governor = governor , cwd = str (workspace ),
176- runner = runner , input = full if spec .provider == "codex" else None )
177- text , usage = _cli_result (spec , getattr (proc , "stdout" , "" ) or "" )
178- error = "" if text else f"empty judge reply (rc={ getattr (proc , 'returncode' , '?' )} )"
179- except JudgeError as exc :
180- text , usage , error = "" , {}, str (exc )
181- finally :
182- shutil .rmtree (workspace , ignore_errors = True )
206+ if spec .provider == "zcode" :
207+ text , usage , error , served = _run_zcode (spec , system = system , prompt = prompt ,
208+ role = role , governor = governor )
209+ else :
210+ workspace = Path (tempfile .mkdtemp (prefix = "judge-ws-" ))
211+ workspace .chmod (0o700 )
212+ try :
213+ full = prompt if not system else f"{ system } \n \n { prompt } "
214+ proc = governed_subprocess (_cli_argv (spec , full , workspace ), governor = governor , cwd = str (workspace ),
215+ runner = runner , input = full if spec .provider == "codex" else None )
216+ text , usage = _cli_result (spec , getattr (proc , "stdout" , "" ) or "" )
217+ error = "" if text else f"empty judge reply (rc={ getattr (proc , 'returncode' , '?' )} )"
218+ except JudgeError as exc :
219+ text , usage , error = "" , {}, str (exc )
220+ finally :
221+ shutil .rmtree (workspace , ignore_errors = True )
222+ served = ""
183223 store = current_store ()
184224 if store is not None :
185225 try :
186226 store .append ("model_call" , inputs = {"system" : system , "prompt" : prompt },
187227 outputs = {"reply" : text } if text else {},
188- model = {"role" : role , "provider" : spec .provider , "model" : spec .model , "served_model" : "" },
228+ model = {"role" : role , "provider" : spec .provider , "model" : spec .model ,
229+ "served_model" : served },
189230 usage = usage , seconds = round (time .monotonic () - started , 3 ),
190231 result = {"format" : "prompt/1" , "usd" : 0.0 , "subscription" : True }, error = error )
191232 except Exception : # noqa: BLE001 - recording never changes a verdict
@@ -197,14 +238,16 @@ def run_judge(spec: JudgeSpec, *, system: str, prompt: str, llm: Any = None, gov
197238
198239def judge_spec_from (settings : Any , override : str = "" ) -> JudgeSpec | None :
199240 """The gold_match / paired judge from ``settings.improve_judge`` (or an
200- explicit override): ``api:<model>`` or ``cli:<provider>:<model>``; None
201- when unset."""
241+ explicit override): ``api:<model>`` or ``cli:<provider>:<model>`` where
242+ the provider is cursor | codex | claude | zcode; None when unset."""
202243 raw = str (override or getattr (settings , "improve_judge" , "" ) or "" )
203244 if not raw :
204245 return None
205246 parts = raw .split (":" )
206247 if parts [0 ] == "api" and len (parts ) == 2 :
207248 return JudgeSpec ("api" , parts [1 ])
208249 if parts [0 ] == "cli" and len (parts ) == 3 :
250+ if parts [1 ] not in ("cursor" , "codex" , "claude" , "zcode" ):
251+ raise JudgeError (f"unknown CLI judge provider { parts [1 ]!r} " )
209252 return JudgeSpec ("cli" , parts [2 ], provider = parts [1 ])
210253 raise JudgeError (f"improve_judge must be api:<model> or cli:<provider>:<model>, got { raw !r} " )
0 commit comments