From bff1c3faeadaf6bead991e79db56cdaa9ddede95 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 13:18:10 +0000 Subject: [PATCH 01/42] fix(repo): Claude Code Bash reads count as reads; bash= reach scenario Settles two points of an audit of a `dokimi repo run` replay on IntelPython/dpnp#3099. Item 6: under Claude Code, guidance_read counted the Read tool only. In the replay 17 of 78 guidance cells printed an AGENTS.md with Bash (`cd benchmarks; cat AGENTS.md`) and all had guidance_read []. The claude-stream-json converter now runs cli_traces.shell_reads on each Bash command, as the Codex and Copilot readers do since #116, and emits a read event per printed file before the command event (same t, the subagent's agent id). Claude Code keeps the shell's directory between calls, so after an agent's call that ran a cd or pushd (shell_reads_moved), its later relative reads are skipped and absolute ones kept. On the tool_result, a hook or permission denial marks the reads denied, an is_error result of a one-file, one-part read drops it (cli_traces._failed_read), and otherwise the reads get rc 0. The new logic sits in helpers, so events._apply drops from complexity 17 to 14. process_metrics is unchanged; the contract (4.1, 6, 7, 7.1) and the module docstrings say which shell reads count, now for all three harnesses. Item 7: no reach scenario measured whether a Bash read of a nested file loads its directory's guidance. `repo reach --scenarios bash` adds bash=, started at the root with the profile's new `bash` tool mode (claude-code: --tools Bash), telling the agent to `cat /REACH_PROBE.txt`. The default kinds stay root,cwd,read. A profile without a bash mode (codex, copilot) is a usage error, exit 2. harness_profile.v1 accepts the optional `tools.bash` key in place; it is left out of delegation.none, whose propertyNames schema_compat holds fixed. The receipt maps bash= as it maps read=. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 8 + ...2_120100_antonwolfy_reach-bash-scenario.md | 8 + dokimi/docs/repo-evaluation-contract.md | 35 ++-- dokimi/docs/reviewing.md | 2 +- dokimi/dokimi/commands/repo.py | 11 +- dokimi/dokimi/repos/cli_traces.py | 13 +- dokimi/dokimi/repos/events.py | 105 ++++++++--- dokimi/dokimi/repos/process_metrics.py | 10 +- dokimi/dokimi/repos/profiles/claude-code.yaml | 1 + dokimi/dokimi/repos/reach.py | 29 ++- dokimi/dokimi/repos/receipt.py | 2 +- .../schemas/effective_instructions.v1.json | 2 +- dokimi/dokimi/schemas/harness_profile.v1.json | 1 + dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_batch_ledger_golden.py | 1 + dokimi/tests/test_repo_events_bash.py | 134 ++++++++++++++ dokimi/tests/test_repo_reach_bash.py | 165 ++++++++++++++++++ 17 files changed, 474 insertions(+), 55 deletions(-) create mode 100644 dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md create mode 100644 dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md create mode 100644 dokimi/tests/test_repo_events_bash.py create mode 100644 dokimi/tests/test_repo_reach_bash.py diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md new file mode 100644 index 000000000..43af0d65c --- /dev/null +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -0,0 +1,8 @@ + + +### Changed + +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd`, that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md new file mode 100644 index 000000000..ecec5a37b --- /dev/null +++ b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md @@ -0,0 +1,8 @@ + + +### Added + +- **A `bash=` reach scenario** — `repo reach --scenarios bash` runs, for each guidance directory, a scenario started at the root with the profile's new `bash` tool mode (`claude-code`: `--tools Bash`) that tells the agent to `cat /REACH_PROBE.txt` and then list tokens: it shows whether a shell read of a nested file loads that directory's guidance, as `read=` shows for the Read tool. The default stays `root,cwd,read`, so a probe costs no more unless asked; `--dry-run` lists the bash scenarios and `--only bash=` picks one. A profile without a `bash` tool mode (`codex`, `copilot`) asked for `--scenarios bash` is a usage error (exit 2) naming the profile. `harness_profile.v1` accepts an optional `bash` key under `tools` (not under `delegation.none`, so `repo run --tools bash --delegation none` is refused); the receipt treats a `bash=` run as it does `read=`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 6377283ab..8df24ef2a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -462,11 +462,11 @@ parent's and subagents' together, so a grader need not: | Key | Meaning | |---|---| -| `tool_calls` | every event: each tool call the agent made | +| `tool_calls` | every event: each tool call the agent made, and the `read` event of each file a shell command printed (see `guidance_read`) | | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events (the Read tool only, in a Claude Code trace), neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. Guidance opened through Bash (`cat AGENTS.md`) is not counted, so `[]` means no Read-tool read; to count it, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd` (the shell keeps its directory between calls, so the file is not known); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -874,6 +874,7 @@ invoke: ["{bin}", "-p", "{prompt}", "--model", "{model}", "--output-format", "st tools: # what the "{tools}" element expands to none: ["--tools", ""] read: ["--tools", "Read"] + bash: ["--tools", "Bash"] # optional: repo reach --scenarios bash (§7) all: ["--disallowedTools", "WebFetch", "WebSearch"] delegation: # --delegation none: these replace the tool mode's arguments none: @@ -939,7 +940,8 @@ args_if_present: # argv added after the "{tools}" element wh `claude-stream-json`, `codex-json` (`codex exec --json`), `copilot-json` (`copilot -p --output-format json`) or `text` (stdout is the answer, no events). Codex has no read tool, so its `read` events come from shell - commands that only print files (`cat`, `head`, `tail`, `sed -n`, `nl`). + commands that only print files (`cat`, `head`, `tail`, `sed -n`, `nl`); + Copilot's shell commands and Claude Code's Bash calls give them too. Neither Codex nor Copilot reports a cost: `cost_usd` is `null` and a batch budget counts each cell's cap. - `bin_companions` names files the executable starts from its own directory. @@ -1029,6 +1031,13 @@ Protocol: - `cwd=`: started in each guidance directory, tools off. - `read=`: started at the root, read tool on. The agent is told to read `/REACH_PROBE.txt`, then list tokens. + - `bash=`: started at the root, the profile's `bash` tool mode (Claude + Code: the Bash tool alone). The agent is told to run + `cat /REACH_PROBE.txt`, then list tokens: it shows whether a shell + read of a file in `` loads that directory's guidance. Only with `--scenarios bash` + (the default is `root,cwd,read`); a profile without a `bash` tool mode + (Codex and Copilot, whose shell commands are already their reads) is a + usage error (exit 2) naming the profile. - A scenario that runs past `--timeout` is an `error`; its raw stdout up to the kill is kept in the trace. Its harness is killed with its process group (POSIX) or job object (Windows), and so is anything left in that @@ -1061,11 +1070,13 @@ per scenario). - `loaded`: both tokens reported; - `truncated`: one token reported; - `fetched`: reported only after the agent read the file with a tool. For the - Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, - `tail`, `nl` or `sed -n`; for Codex, a command that is one such read of one - file and exits non-zero is not a read (the exit code of `cat a missing` does - not say which file failed, so both stay reads). A read through a pipe, `grep` - or `rg` is not seen, so such a file can be classed `loaded`; + Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts + only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one + such read of one file and exits non-zero is not a read (the exit code of + `cat a missing` does not say which file failed, so both stay reads). A read + through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative + read after an earlier Bash call of the same agent ran a `cd`, so such a file + can be classed `loaded`; - `not_loaded`; - `error`: the scenario produced no usable answer: no answer, a non-zero exit, or an error result. @@ -1120,7 +1131,7 @@ Each source carries: | `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | | `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | | `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | -| `states.accessed` | the harness's read events | `true` if the harness reported a read of the file (the Read tool; Codex and Copilot also report supported shell reads, §7); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | +| `states.accessed` | the harness's read events | `true` if the trace has a read event of the file (a read tool, or a shell read §7 lists, under all three harnesses); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | | `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | | `states.loaded` | the class | `true` for loaded, `false` for not_loaded, `unknown` for fetched, truncated and error: a tool read is not proof that start-up loading did not happen | | `states.followed` | | always `unknown` (the schema's `const`) | @@ -1161,9 +1172,9 @@ reported in a run, so it can support `loaded`, `fetched` and `not_loaded` as observations of that run. It cannot show: - that an instruction was followed (`followed` is always `unknown`); -- a read through a shell that the harness does not normalise into a read - event (`cat`, `rg`, `grep` under Claude Code; Codex and Copilot normalise - only the shell commands §7 lists); +- a shell read that the converter does not turn into a read event: under + Claude Code, Codex and Copilot alike, only the shell commands §7 lists are + read events, so `rg`, `grep` or a pipe stays unseen; - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: they appear with `observed: unknown`. A skill body is loaded on demand, so it diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 6fa1a3ff1..8030f8806 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -102,7 +102,7 @@ The full path with reading notes: [skill-novelty-walkthrough.md](skill-novelty-w | Step | Command | Cost | |---|---|---| | 1. Do the paths, links and commands named in the guidance exist? | `dokimi repo scan ` | free | -| 2. Which guidance files does the harness actually load? | `dokimi repo reach --model --dry-run`, then a few `--only ` with `--budget` | capped by `--budget` (default $0.10) per scenario; needs the `claude` binary and `ANTHROPIC_API_KEY`, or `AWS_BEARER_TOKEN_BEDROCK` with `AWS_REGION` and `CLAUDE_CODE_USE_BEDROCK` | +| 2. Which guidance files does the harness actually load? | `dokimi repo reach --model --dry-run`, then a few `--only ` with `--budget`; to see whether a Bash read of a nested file loads its directory's guidance (agents that read with `cat`), `--scenarios bash` (or `--scenarios bash --only bash=`) | capped by `--budget` (default $0.10) per scenario; needs the `claude` binary and `ANTHROPIC_API_KEY`, or `AWS_BEARER_TOKEN_BEDROCK` with `AWS_REGION` and `CLAUDE_CODE_USE_BEDROCK` | | 3. What does the root guidance add? | `dokimi novelty analyze --artifact text:/AGENTS.md --tier triage …` | ~100 calls | | 4. Does the guidance change task outcomes? | `dokimi repo run --manifest-file … --model --batch --arm guidance:none,guidance:raw --agent-network host --budget --max-batch-usd ` | needs the repository's own tasks, Docker, `claude` binary on PATH (or `--bin`); `--budget` defaults to $2.00 per agent run with no batch cap unless you pass `--max-batch-usd` | diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 9f61497ca..40a773a1d 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -242,12 +242,12 @@ def cmd_repo_fetch(args: argparse.Namespace) -> None: def _check_reach_flags(args: argparse.Namespace, kinds: tuple[str, ...]) -> None: """Refuse a flag that would drop a scenario kind silently, or spend a budget that is not one.""" - from dokimi.repos.reach import ReachError + from dokimi.repos.reach import KINDS, ReachError # A typo would otherwise drop a scenario kind silently and still spend the budget. - unknown_kinds = sorted(set(kinds) - {"root", "cwd", "read"}) + unknown_kinds = sorted(set(kinds) - set(KINDS)) if unknown_kinds: - raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: root, cwd, read)") + raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: {', '.join(KINDS)})") if not math.isfinite(args.budget) or args.budget <= 0: raise ReachError(f"--budget must be > 0, got {args.budget:g}") if args.model is None: @@ -373,6 +373,7 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: from dokimi.repos.source import SourceArgumentError from dokimi.repos.reach import ( ReachError, + check_kinds, planned, probe_files, run_reach, @@ -385,6 +386,7 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: try: # the arguments as typed: a usage error _check_reach_flags(args, kinds) profile = load_profile(args.harness) + check_kinds(profile, kinds) source = RepoSource.parse(args.source) except (SourceError, ProfileError, ReachError) as exc: print(f"✗ {exc}", file=sys.stderr) @@ -686,7 +688,8 @@ def register(sub, positive_int) -> None: reach_p.add_argument("--bin", default=None, help="Harness executable (default: the profile's `bin`)") reach_p.add_argument("--out", default=None, help="Output directory: reach.json and traces/ (default: reach-out)") reach_p.add_argument("--scenarios", default=None, - help="Comma list of scenario kinds: root, cwd, read (default: all)") + help="Comma list of scenario kinds: root, cwd, read, bash (default: root,cwd,read; bash " + "cats each probe file through the Bash tool, for a profile with a bash tool mode)") reach_p.add_argument("--only", action="append", default=None, metavar="SCENARIO", help="Run only this scenario id (for example read=cpp); repeatable") reach_p.add_argument("--work-dir", default=None, dest="work_dir", diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 8233e1f55..b9a7a32d4 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -96,6 +96,17 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: redirected to a file or /dev/null is dropped, as that output is not shown to the agent. ``--help`` and ``--version`` print and open no file. A function definition is skipped whole. Absolute paths are kept. """ + return _walk_script(script, cwd)[0] + + +def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: + """:func:`shell_reads`, and whether the script may leave its shell in another directory: it ran a ``cd``, + ``pushd`` or ``popd`` outside a subshell, or a ``cd`` whose target is not known.""" + reads, walk = _walk_script(script, cwd) + return reads, walk.moved + + +def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: out: list[str] = [] walk = _Walk(posixpath.normpath(cwd) if cwd else "") previous = "" @@ -121,7 +132,7 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: words = walk.words(part) if words: out.extend(walk.command(words, after, chained)) - return out + return out, walk class _Walk: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 12a0ceda0..8bae9b747 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -18,6 +18,11 @@ bytes, and ``output_truncated`` says whether that happened. An event a subagent made carries ``agent``: the id of the parent's tool call that started it. +A shell command that only prints files (:func:`~dokimi.repos.cli_traces.shell_reads`) also gives a +``read`` event per file, before its ``command`` event: a Codex or Copilot shell command, and +Claude Code's ``Bash`` tool. Claude Code keeps the shell's directory from one ``Bash`` call to the +next, so once a call of an agent ran a ``cd``, that agent's later relative reads are not recorded. + Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` (parent, subagents and helper models) when the result has it, else the parent's ``usage``; ``usage_scope`` says which (``tree`` or ``main``). @@ -27,16 +32,19 @@ import json import re +from dataclasses import dataclass, field from datetime import datetime from typing import Any from dokimi.context.trace import parse_trace +from .cli_traces import _failed_read, codex_json, copilot_json, shell_reads_moved from .transcript import Transcript, _parse_time, clip_output, relative_path TRACE_FORMATS = ("claude-stream-json", "codex-json", "copilot-json", "text") -_COMMAND_TOOLS = {"Bash", "PowerShell"} +_SHELL_TOOL = "Bash" # a POSIX shell: its command is parsed for the files it prints (not PowerShell's) +_COMMAND_TOOLS = {_SHELL_TOOL, "PowerShell"} _READ_TOOLS = {"Read"} _EDIT_TOOLS = {"Edit", "MultiEdit", "Write", "NotebookEdit"} _AGENT_TOOLS = {"Agent", "Task"} @@ -47,22 +55,32 @@ def convert(fmt: str, raw: str, workdir: str | None = None, cwd: str = "") -> Tr """Convert a raw harness stdout in format ``fmt`` to a :class:`Transcript`. ``cwd`` is the directory the harness started in, relative to ``workdir``: a shell command's relative - paths (Codex, Copilot) are read from there.""" + paths are read from there (in Claude Code, every agent's first ``Bash`` call starts there).""" if fmt == "text": return Transcript(answer=raw.strip()) if fmt == "claude-stream-json": - return _claude_stream_json(raw, workdir) + return _claude_stream_json(raw, workdir, cwd) if fmt in ("codex-json", "copilot-json"): - from .cli_traces import codex_json, copilot_json - return (codex_json if fmt == "codex-json" else copilot_json)(raw, workdir, cwd) raise ValueError(f"unknown trace format {fmt!r} (known: {', '.join(TRACE_FORMATS)})") -def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: +@dataclass +class _Stream: + """What a Claude stream-json trace carries from line to line.""" + + workdir: str | None + cwd: str # where each agent's shell starts, relative to workdir + by_id: dict[str, dict[str, Any]] = field(default_factory=dict) # tool_use id -> its event + # tool_use id of a Bash call -> the read events derived from it, and the files its command names + shell_reads: dict[str, tuple[list[dict[str, Any]], list[str]]] = field(default_factory=dict) + moved: set[str] = field(default_factory=set) # agents ("" the parent) whose shell may have left cwd + + +def _claude_stream_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: out = Transcript() start: datetime | None = None - by_id: dict[str, dict[str, Any]] = {} # tool_use id -> its event + stream = _Stream(workdir, cwd) for n, line in enumerate(raw.split("\n"), 1): # not splitlines: a JSON string may hold U+2028 or U+0085 if not line.strip(): continue @@ -79,7 +97,7 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: start = start or stamp t = round((stamp - start).total_seconds(), 3) try: - _apply(out, rec, t, by_id, workdir) + _apply(out, rec, t, stream) except (AttributeError, TypeError) as exc: # valid JSON of an unexpected shape out.errors.append(f"line {n}: unexpected shape ({exc})") main = parse_trace(raw).main_calls() @@ -87,9 +105,10 @@ def _claude_stream_json(raw: str, workdir: str | None) -> Transcript: return out -def _apply_result(block: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> None: +def _apply_result(out: Transcript, block: dict[str, Any], stream: _Stream) -> None: """Attach one tool_result to the event of its tool_use.""" - result_of = by_id.get(block.get("tool_use_id", "")) + tool_id = block.get("tool_use_id", "") + result_of = stream.by_id.get(tool_id) if result_of is None: return output = _result_text(block.get("content")) @@ -97,7 +116,9 @@ def _apply_result(block: dict[str, Any], by_id: dict[str, dict[str, Any]]) -> No # naming the hook; the command never ran, so it has no rc. denied = bool(block.get("is_error")) and bool(_HOOK_DENIAL.match(output or "")) if denied: - result_of["denied"] = True + _deny(stream, tool_id) + else: + _settle_shell_reads(out, stream, tool_id, bool(block.get("is_error"))) if result_of["kind"] == "read" and not denied: # A read that failed (the file does not exist) shows nothing of the file. result_of["rc"] = 1 if block.get("is_error") else 0 @@ -115,8 +136,51 @@ def _set_num_turns(out: Transcript, turns: Any) -> None: out.errors.append(f"result num_turns {str(turns)[:50]!r} is not an integer; recorded as null") -def _apply(out: Transcript, rec: dict[str, Any], t: float | None, - by_id: dict[str, dict[str, Any]], workdir: str | None) -> None: +def _deny(stream: _Stream, tool_id: Any) -> None: + """A refused call: its event and the reads derived from it, which never ran.""" + stream.by_id[tool_id]["denied"] = True + for read in stream.shell_reads.get(tool_id, ([], []))[0]: + read["denied"] = True + + +def _settle_shell_reads(out: Transcript, stream: _Stream, tool_id: Any, is_error: bool) -> None: + """The reads of a Bash call that ran: dropped when it failed and was one read of one file (``cat AGENTS.md`` of + a missing file), else ``rc`` 0, since a failure does not say which file failed (``cat a missing``).""" + reads, named = stream.shell_reads.get(tool_id, ([], [])) + if is_error and _failed_read(str(stream.by_id[tool_id].get("text", "")), named, 1): + dropped = {id(e) for e in reads} + out.events[:] = [e for e in out.events if id(e) not in dropped] + return + for read in reads: + read["rc"] = 0 + + +def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: + """The events of one tool_use block: a Bash call's derived reads, then its own event.""" + event = _tool_event(block, t, stream.workdir) + reads, named = _bash_reads(event, agent, stream) if block.get("name") == _SHELL_TOOL else ([], []) + if agent: + for e in (*reads, event): + e["agent"] = agent + if block.get("name") in _AGENT_TOOLS: + out.subagent_calls = (out.subagent_calls or 0) + 1 + out.events.extend([*reads, event]) + if block.get("id"): + stream.by_id[block["id"]] = event + stream.shell_reads[block["id"]] = (reads, named) + + +def _bash_reads(event: dict[str, Any], agent: str, stream: _Stream) -> tuple[list[dict[str, Any]], list[str]]: + """The read events of a Bash command and the files it names. Once an earlier call of the same agent may have + left the start directory, a relative path names no known file: only absolute ones are kept.""" + named, moves = shell_reads_moved(str(event["text"]), stream.cwd) + known = [p for p in named if p.startswith("/")] if agent in stream.moved else named + if moves: + stream.moved.add(agent) + return [{"t": event["t"], "kind": "read", "path": relative_path(p, stream.workdir)} for p in known], named + + +def _apply(out: Transcript, rec: dict[str, Any], t: float | None, stream: _Stream) -> None: kind = rec.get("type") if kind == "system" and rec.get("subtype") == "init": out.harness_version = rec.get("claude_code_version") @@ -127,14 +191,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "assistant": for block in _content(rec): if block.get("type") == "tool_use": - event = _tool_event(block, t, workdir) - if rec.get("parent_tool_use_id"): - event["agent"] = str(rec["parent_tool_use_id"]) - if block.get("name") in _AGENT_TOOLS: - out.subagent_calls = (out.subagent_calls or 0) + 1 - out.events.append(event) - if block.get("id"): - by_id[block["id"]] = event + _tool_use(out, block, t, str(rec.get("parent_tool_use_id") or ""), stream) model = (rec.get("message") or {}).get("model") # A subagent's model is its definition's, not the run's; "" marks a reply Claude Code wrote itself. if model and model != "" and rec.get("parent_tool_use_id") is None: @@ -142,7 +199,7 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, elif kind == "user": for block in _content(rec): if block.get("type") == "tool_result": - _apply_result(block, by_id) + _apply_result(out, block, stream) elif kind == "result": out.answer = str(rec.get("result") or "").strip() out.cost_usd = rec.get("total_cost_usd") @@ -159,8 +216,8 @@ def _apply(out: Transcript, rec: dict[str, Any], t: float | None, out.usage_scope, (out.input_tokens, out.output_tokens, out.cache_read_tokens, out.cache_write_tokens) = counts for denial in rec.get("permission_denials") or []: - if isinstance(denial, dict) and denial.get("tool_use_id") in by_id: - by_id[denial["tool_use_id"]]["denied"] = True + if isinstance(denial, dict) and denial.get("tool_use_id") in stream.by_id: + _deny(stream, denial["tool_use_id"]) def _tool_event(block: dict[str, Any], t: float | None, workdir: str | None) -> dict[str, Any]: diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index f0a0fdd94..331669a22 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -5,14 +5,16 @@ The framework counts what every repository asks of a trace: -- ``tool_calls``: every event, so every tool call of the parent and its subagents; +- ``tool_calls``: every event, so every tool call of the parent and its subagents, and the ``read`` event of + each file a shell command printed; - ``commands``: the ``command`` events; - ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means no result came back, which is not a failure; a permission denial carries ``rc`` 1 but never ran); - ``denied_commands``: commands a hook or the harness's permissions refused (``denied``); -- ``guidance_read``: the sorted workspace paths of ``read`` events (the Read tool only; a Bash ``cat`` is not - counted, a ``process_metrics`` counter can) not denied that the manifest's guidance globs match; a path - outside the workspace (absolute, or with ``..``) is no repository file. +- ``guidance_read``: the sorted workspace paths of ``read`` events (a read tool, or a shell command that only + prints files, ``cat`` / ``head`` / ``tail`` / ``nl`` / ``sed -n``; a pipe, ``grep`` or ``rg`` is not one) not + denied that the manifest's guidance globs match; a path outside the workspace (absolute, or with ``..``) is + no repository file. ``counters`` holds the manifest's ``process_metrics``: for each name, how many ``command`` events its regex finds a match in (``re.search``). A counter matches the command text unless it names diff --git a/dokimi/dokimi/repos/profiles/claude-code.yaml b/dokimi/dokimi/repos/profiles/claude-code.yaml index 54fdf12b2..a5853ab45 100644 --- a/dokimi/dokimi/repos/profiles/claude-code.yaml +++ b/dokimi/dokimi/repos/profiles/claude-code.yaml @@ -28,6 +28,7 @@ invoke: tools: none: ["--tools", ""] read: ["--tools", "Read"] + bash: ["--tools", "Bash"] # repo reach --scenarios bash: what a shell read of a nested file loads all: ["--disallowedTools", "WebFetch", "WebSearch"] # --delegation none: the same tools without the ones that start subagents (Agent/Task, Workflow). delegation: diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 98a0ea65f..13cd8f026 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -13,6 +13,9 @@ - ``cwd=``: started in each directory that holds guidance, tools disabled - ``read=``: started at the root, told to read one file in ```` with the read tool and then list tokens (finds lazy, subtree-triggered loading) +- ``bash=``: as ``read=``, but the file is printed with ``cat`` through + the Bash tool (whether a shell read triggers that loading); only when asked for, + and only for a profile with a ``bash`` tool mode A file is ``loaded`` when both tokens are reported, ``truncated`` when only one is, ``fetched`` when it was reported only after the agent read the file @@ -73,12 +76,22 @@ "and nothing else. Report only what is already in your context; do not search for " "files. If there are none, answer NONE.") ASK_AFTER_READ = "First use the Read tool to read the file {path}, and read no other file. Then: " +ASK_AFTER_BASH = "First use the Bash tool to run `cat {path}`, and run no other command. Then: " +KINDS = ("root", "cwd", "read", "bash") # the scenario kinds; bash only when asked for +DEFAULT_KINDS = ("root", "cwd", "read") class ReachError(RuntimeError): """The probe cannot run safely (for example a guidance file above the workspace).""" +def check_kinds(profile: HarnessProfile, kinds: tuple[str, ...]) -> None: + """Refuse a ``bash`` scenario for a profile without a ``bash`` tool mode, before any scenario runs.""" + if "bash" in kinds and "bash" not in profile.tools: + raise ReachError(f"profile {profile.name} has no `bash` tool mode, so it cannot run the bash scenarios " + f"(its tool modes: {', '.join(sorted(profile.tools))})") + + @dataclass(frozen=True) class SeededFile: path: str @@ -93,11 +106,13 @@ class Scenario: id: str cwd: str # relative to the workspace root; "" is the root tools: str - read: str | None = None # file the agent is told to read first + read: str | None = None # file the agent is told to read first (with ``cat`` in the bash tool mode) @property def prompt(self) -> str: - return (ASK_AFTER_READ.format(path=self.read) if self.read else "") + ASK + if not self.read: + return ASK + return (ASK_AFTER_BASH if self.tools == "bash" else ASK_AFTER_READ).format(path=self.read) + ASK def token(salt: str, path: str, position: str) -> str: @@ -116,7 +131,7 @@ def probe_files(tree: RepoTree, manifest: RepoManifest, profile: HarnessProfile) return sorted(p for p in declared | native if not manifest.in_eval_dir(p)) -def scenarios(files: list[str], kinds: tuple[str, ...] = ("root", "cwd", "read"), +def scenarios(files: list[str], kinds: tuple[str, ...] = DEFAULT_KINDS, only: tuple[str, ...] = ()) -> list[Scenario]: dirs = sorted({str(PurePosixPath(f).parent) for f in files} - {"."}) out: list[Scenario] = [] @@ -124,8 +139,9 @@ def scenarios(files: list[str], kinds: tuple[str, ...] = ("root", "cwd", "read") out.append(Scenario("cwd=.", "", "none")) if "cwd" in kinds: out.extend(Scenario(f"cwd={d}", d, "none") for d in dirs) - if "read" in kinds: - out.extend(Scenario(f"read={d}", "", "read", read=f"{d}/{PROBE_FILE}") for d in dirs) + for kind in ("read", "bash"): # started at the root, told to read the probe file of each directory + if kind in kinds: + out.extend(Scenario(f"{kind}={d}", "", kind, read=f"{d}/{PROBE_FILE}") for d in dirs) return [s for s in out if s.id in only] if only else out @@ -285,7 +301,7 @@ def run_reach( out_dir: Path, bin_path: str | None = None, work_dir: Path | None = None, - kinds: tuple[str, ...] = ("root", "cwd", "read"), + kinds: tuple[str, ...] = DEFAULT_KINDS, only: tuple[str, ...] = (), salt: str | None = None, max_turns: int = 4, @@ -294,6 +310,7 @@ def run_reach( keep: bool = False, ) -> dict[str, Any]: """Run every scenario once and return the guidance_reach.v1 payload (unsaved).""" + check_kinds(profile, kinds) files = probe_files(tree, manifest, profile) if not files: raise ReachError("no guidance files to probe: the manifest's globs and the " diff --git a/dokimi/dokimi/repos/receipt.py b/dokimi/dokimi/repos/receipt.py index 0b60ca44a..40545ef43 100644 --- a/dokimi/dokimi/repos/receipt.py +++ b/dokimi/dokimi/repos/receipt.py @@ -226,7 +226,7 @@ def _budget_only(path: str, run: _Run) -> dict[str, Any]: def _states(cls: str | None, tools_off: bool, read_by_tool: bool) -> dict[str, Any]: - """accessed: a Read-tool read was seen (false with no tools at all); selected: the probe saw the source enter the + """accessed: a read event was seen, by a read tool or a shell read (false with no tools at all); selected: the probe saw the source enter the request, by the harness or by the agent's own read; loaded: at the harness's own load step, not through a read.""" accessed: bool | str = True if read_by_tool else (False if tools_off else UNKNOWN) selected: bool | str = True if cls in ("loaded", "fetched") else UNKNOWN diff --git a/dokimi/dokimi/schemas/effective_instructions.v1.json b/dokimi/dokimi/schemas/effective_instructions.v1.json index 7e79f6066..87c269b4a 100644 --- a/dokimi/dokimi/schemas/effective_instructions.v1.json +++ b/dokimi/dokimi/schemas/effective_instructions.v1.json @@ -40,7 +40,7 @@ "run_id": {"type": "string", "description": "The reach salt and the scenario id, so the receipt names its record in reach.json."}, "runtime": {"$ref": "#/$defs/runtime"}, "cwd": {"type": "string", "description": "The directory the run started in; `.` is the repository root."}, - "target": {"type": ["string", "null"], "description": "The file the run was told to read (the probe file of a read= scenario), else null."}, + "target": {"type": ["string", "null"], "description": "The file the run was told to read (the probe file of a read= or bash= scenario), else null."}, "sources": {"type": "array", "items": {"$ref": "#/$defs/source"}}, "precedence_observed": { "type": "array", diff --git a/dokimi/dokimi/schemas/harness_profile.v1.json b/dokimi/dokimi/schemas/harness_profile.v1.json index 956b85b2d..2a2e9b839 100644 --- a/dokimi/dokimi/schemas/harness_profile.v1.json +++ b/dokimi/dokimi/schemas/harness_profile.v1.json @@ -19,6 +19,7 @@ "properties": { "none": {"type": "array", "items": {"type": "string"}}, "read": {"type": "array", "items": {"type": "string"}}, + "bash": {"type": "array", "items": {"type": "string"}, "description": "The shell tool alone: `repo reach --scenarios bash`. A profile whose shell commands already give its read events (Codex, Copilot) needs none."}, "all": {"type": "array", "items": {"type": "string"}} } }, diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 430236aa1..b2af48562 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -76,7 +76,7 @@ "dokimi/repos/environment/risks.py::_runs_repository_code": {"complexity": 12, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::_segments": {"complexity": 29, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/environment/risks.py::detect_capabilities": {"complexity": 16, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/repos/events.py::_apply": {"complexity": 17, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/repos/events.py::_apply": {"complexity": 14, "rationale": "pre-existing when the gate was introduced"}, "dokimi/repos/harness_profile.py::parse_profile": {"complexity": 11}, "dokimi/repos/l0/adapters.py::_build_names": {"complexity": 15, "rationale": "grew on main in #115 (L0 rules for static_check parity) before the ratchet landed; split it in its own PR"}, "dokimi/repos/l0/adapters.py::_make": {"complexity": 11}, diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index df3069a53..081ae2e85 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -24,6 +24,7 @@ GOLDEN_CLAUDE_CODE_IDS = { ("all", "native"): "e6eabec71f42", ("all", "none"): "2324b5c59f99", + ("bash", "native"): "ebecf5b62ae8", ("none", "native"): "7d8aa618677b", ("none", "none"): "7d8aa618677b", ("read", "native"): "e9dbd28050f2", diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py new file mode 100644 index 000000000..3834864c9 --- /dev/null +++ b/dokimi/tests/test_repo_events_bash.py @@ -0,0 +1,134 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Claude Code ``Bash`` calls that print files give ``read`` events, as Codex and Copilot shell commands do.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.process_metrics import process_metrics + +WS = "/ws" + + +def _call(tool_id: str, command: str, agent: str | None = None, name: str = "Bash") -> dict: + rec = {"type": "assistant", "timestamp": "2026-01-01T00:00:05Z", "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": command}}]}} + return {**rec, "parent_tool_use_id": agent} if agent else rec + + +def _result(tool_id: str, text: str = "ok", is_error: bool = False) -> dict: + return {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": is_error, "content": text}]}} + + +def _events(*lines: dict, cwd: str = "") -> list[dict]: + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + return convert("claude-stream-json", "\n".join(json.dumps(x) for x in (first, *lines)), WS, cwd).events + + +def _reads(events: list[dict]) -> list[str]: + return [e["path"] for e in events if e["kind"] == "read"] + + +def test_a_cd_then_cat_in_one_call_reads_the_file_in_that_directory(): + events = _events(_call("a", "cd benchmarks; cat AGENTS.md"), _result("a", "# rules")) + assert events == [ + {"t": 5.0, "kind": "read", "path": "benchmarks/AGENTS.md", "rc": 0}, + {"t": 5.0, "kind": "command", "text": "cd benchmarks; cat AGENTS.md", "rc": 0, "output": "# rules", + "output_truncated": False}, + ] + + +def test_an_absolute_path_inside_the_workspace_is_made_relative(): + assert _reads(_events(_call("a", "head -n 40 /ws/sub/AGENTS.md"), _result("a"))) == ["sub/AGENTS.md"] + + +def test_a_failed_cat_of_one_file_is_no_read(): + events = _events(_call("a", "cat AGENTS.md"), _result("a", "cat: AGENTS.md: No such file or directory", True)) + assert events == [{"t": 5.0, "kind": "command", "text": "cat AGENTS.md", "rc": 1, + "output": "cat: AGENTS.md: No such file or directory", "output_truncated": False}] + + +def test_a_failed_cat_of_two_files_keeps_both_reads(): + events = _events(_call("a", "cat AGENTS.md missing.md"), _result("a", "# rules\ncat: missing.md: No such file", True)) + assert [(e["path"], e["rc"]) for e in events if e["kind"] == "read"] == [("AGENTS.md", 0), ("missing.md", 0)] + + +@pytest.mark.parametrize("refusal", [ + lambda: _result("a", "PreToolUse:Bash hook error: shell background", True), + lambda: {"type": "result", "result": "ok", "permission_denials": [{"tool_name": "Bash", "tool_use_id": "a"}]}, +]) +def test_a_refused_bash_call_marks_its_reads_denied(refusal): + events = _events(_call("a", "cat AGENTS.md"), refusal()) + assert [e.get("denied") for e in events] == [True, True] + assert process_metrics(events, lambda p: p.endswith("AGENTS.md"))["guidance_read"] == [] + + +@pytest.mark.parametrize("command", ["cat AGENTS.md | head", "grep -n rule AGENTS.md", "cat AGENTS.md > copy.md"]) +def test_a_pipe_a_search_or_a_redirect_is_no_read(command): + assert _reads(_events(_call("a", command), _result("a"))) == [] + + +def test_a_powershell_command_is_not_parsed(): + assert _reads(_events(_call("a", "cat AGENTS.md", name="PowerShell"), _result("a"))) == [] + + +def test_after_a_cd_in_an_earlier_call_only_absolute_reads_are_kept(): + events = _events( + _call("a", "cat AGENTS.md"), _result("a"), + _call("b", "cd sub && make"), _result("b"), + _call("c", "cat AGENTS.md"), _result("c"), # in sub, or wherever the cd left the shell: not known + _call("d", "cat /ws/sub/AGENTS.md"), _result("d"), + _call("e", "cd /ws/docs && cat README.md"), _result("e")) # a cd to an absolute path is followed + assert _reads(events) == ["AGENTS.md", "sub/AGENTS.md", "docs/README.md"] + + +@pytest.mark.parametrize("first", ["(cd sub && make)", "make -C sub", "env -C sub make"]) +def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): + events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) + assert _reads(events) == ["AGENTS.md"] + + +def test_a_pushd_in_an_earlier_call_also_loses_the_directory(): + events = _events(_call("a", "pushd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) + assert _reads(events) == [] + + +def test_each_agent_keeps_its_own_shell_directory(): + events = _events( + _call("a", "cd sub"), _result("a"), + _call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), # a subagent's shell starts at the root + _call("p2", "cat AGENTS.md"), _result("p2"), # the parent's shell is in sub: not recorded + _call("s3", "cat AGENTS.md", agent="t1"), _result("s3"), # the parent's cd did not move the subagent + _call("s4", "cd x", agent="t1"), _result("s4"), + _call("s5", "cat AGENTS.md", agent="t1"), _result("s5")) + assert [(e["path"], e.get("agent")) for e in events if e["kind"] == "read"] == [ + ("AGENTS.md", "t1"), ("AGENTS.md", "t1")] + + +def test_a_subagent_bash_read_carries_the_agent(): + events = _events(_call("s1", "cat sub/AGENTS.md", agent="t1"), _result("s1")) + assert [(e["kind"], e.get("agent")) for e in events] == [("read", "t1"), ("command", "t1")] + + +def test_the_shell_starts_in_the_directory_the_harness_started_in(): + assert _reads(_events(_call("a", "cat AGENTS.md"), _result("a"), cwd="sub")) == ["sub/AGENTS.md"] + + +def test_a_bash_call_with_no_result_keeps_its_read_without_an_rc(): + events = _events(_call("a", "cat AGENTS.md")) + assert events[0] == {"t": 5.0, "kind": "read", "path": "AGENTS.md"} + + +def test_guidance_printed_through_bash_is_guidance_read(): + events = _events(_call("b", "cat AGENTS.md"), _result("b", "cat: AGENTS.md: No such file or directory", True), + _call("c", "cat README.md"), _result("c"), + _call("a", "cd benchmarks; cat AGENTS.md"), _result("a")) + got = process_metrics(events, lambda p: p.endswith("AGENTS.md")) + assert got["guidance_read"] == ["benchmarks/AGENTS.md"] + assert got["commands"] == 3 and got["failed_commands"] == 1 diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py new file mode 100644 index 000000000..340577425 --- /dev/null +++ b/dokimi/tests/test_repo_reach_bash.py @@ -0,0 +1,165 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``bash=`` reach scenario: whether a Bash read of a nested file loads its directory's guidance (no LLM). + +The fake harness prints Claude stream-json. With its Bash tool on and a prompt that names a ``cat``, it runs that +``cat`` as a Bash call; ``FAKE_MODE`` says what follows: ``lazy`` loads the file's directory's AGENTS.md as context, +``fetch`` also cats that AGENTS.md itself, ``none`` loads nothing more. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +import yaml + +from dokimi.cli import build_parser +from dokimi.repos import WorkTree, load_manifest +from dokimi.repos.harness_profile import ProfileError, load_profile +from dokimi.repos.reach import ASK, ASK_AFTER_BASH, ReachError, run_reach, scenarios +from dokimi.repos.receipt import build_receipt +from tests.test_repo_reach import _write +from tests.test_repo_receipt import BUDGET, _record, _sources, _tree +from tests.test_repo_receipt import _reach as _saved_reach + +BASH_HARNESS = r''' +import json, os, re, sys +from pathlib import Path + +args = sys.argv[1:] +if args == ["--version"]: + print("9.8.7 (Fake Code)") + sys.exit(0) +mode = os.environ.get("FAKE_MODE", "lazy") +prompt = args[args.index("-p") + 1] +tools = args[args.index("--tools") + 1] +root = Path.cwd() +context = [root / "AGENTS.md"] +events = [{"type": "system", "subtype": "init", "claude_code_version": "9.8.7"}] +def bash(n, command, output): + events.append({"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": f"b{n}", "name": "Bash", "input": {"command": command}}]}}) + events.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": f"b{n}", "is_error": False, "content": output}]}}) +match = re.search(r"run `cat (\S+)`", prompt) +if match and tools == "Bash": + probe = Path(match.group(1)) + bash(1, f"cat {probe}", (root / probe).read_text()) + if mode == "fetch": + bash(2, f"cat {probe.parent}/AGENTS.md", (root / probe.parent / "AGENTS.md").read_text()) + if mode in ("lazy", "fetch"): + context.append(root / probe.parent / "AGENTS.md") +text = "".join(p.read_text() for p in context) +answer = "\n".join(dict.fromkeys(re.findall(r"RC[0-9A-F]{6}", text))) or "NONE" +events.append({"type": "result", "subtype": "success", "is_error": False, "result": answer, + "total_cost_usd": 0.001, "num_turns": 2, "usage": {"input_tokens": 10, "output_tokens": 5}}) +for e in events: + print(json.dumps(e)) +''' + + +def _setup(tmp_path: Path) -> tuple[Path, Path]: + """A repository with a root and a nested AGENTS.md, and a fake profile with a ``bash`` tool mode.""" + root = tmp_path / "repo" + _write(root, "AGENTS.md", "# root\n") + _write(root, "sub/AGENTS.md", "# sub\n") + _write(root, "evals/tasks/.keep", "") + _write(root, "evals/repo-eval.yaml", yaml.safe_dump({ + "schema": "repo_manifest.v1", "id": "toy", "eval_dir": "evals", "guidance": {"globs": ["**/AGENTS.md"]}, + "entry_points": {"grade": "python3 evals/grade.py {run_dir}"}, "tasks": "evals/tasks"})) + script = tmp_path / "bash_harness.py" + script.write_text(BASH_HARNESS, encoding="utf-8") + profile = tmp_path / "fake.yaml" + profile.write_text(yaml.safe_dump({ + "schema": "harness_profile.v1", "name": "fake", "bin": sys.executable, + "version_cmd": ["{bin}", str(script), "--version"], + "invoke": ["{bin}", str(script), "-p", "{prompt}", "--model", "{model}", "{tools}"], + "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "bash": ["--tools", "Bash"]}, + "env_allow": ["PATH", "FAKE_MODE", "LD_LIBRARY_PATH"], "trace": "claude-stream-json", + "guidance": {"names": ["AGENTS.md"]}}), encoding="utf-8") + return root, profile + + +def _probe(tmp_path: Path, monkeypatch, mode: str) -> dict: + monkeypatch.setenv("FAKE_MODE", mode) + root, profile = _setup(tmp_path) + tree = WorkTree(root) + (tmp_path / "work").mkdir() + return run_reach(tree=tree, manifest=load_manifest(tree, "evals/repo-eval.yaml"), profile=load_profile(profile), + model="m", out_dir=tmp_path / "out", work_dir=tmp_path / "work", salt="s", kinds=("bash",)) + + +def test_bash_scenarios_cat_each_probe_file_from_the_root(): + files = ["AGENTS.md", "a/AGENTS.md", "sub/AGENTS.md"] + got = scenarios(files, ("bash",)) + assert [(s.id, s.cwd, s.tools, s.read) for s in got] == [ + ("bash=a", "", "bash", "a/REACH_PROBE.txt"), ("bash=sub", "", "bash", "sub/REACH_PROBE.txt")] + assert got[1].prompt == ("First use the Bash tool to run `cat sub/REACH_PROBE.txt`, and run no other command. " + "Then: " + ASK) + assert got[1].prompt == ASK_AFTER_BASH.format(path="sub/REACH_PROBE.txt") + ASK + assert not [s for s in scenarios(files) if s.tools == "bash"] # the default kinds run no bash scenario + + +def test_the_claude_code_profile_runs_the_bash_mode_with_the_bash_tool_alone(): + profile = load_profile("claude-code") + assert profile.argv(prompt="p", model="m", tools="bash")[-2:] == ["--tools", "Bash"] + with pytest.raises(ProfileError, match="lists no arguments for tool mode 'bash'"): # a reach mode, not a run's + profile.argv(prompt="p", model="m", tools="bash", delegation="none") + + +@pytest.mark.parametrize("harness", ["codex", "copilot"]) +def test_a_profile_without_a_bash_mode_is_a_usage_error(tmp_path, capsys, harness): + root, _ = _setup(tmp_path) + args = build_parser().parse_args(["repo", "reach", str(root), "--manifest", "evals/repo-eval.yaml", "--harness", + harness, "--model", "m", "--scenarios", "root,bash", "--dry-run"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert f"profile {harness} has no `bash` tool mode" in capsys.readouterr().err + + +def test_run_reach_refuses_bash_for_a_profile_without_it_before_any_run(tmp_path): + root, _ = _setup(tmp_path) + tree = WorkTree(root) + with pytest.raises(ReachError, match="profile codex has no `bash` tool mode"): + run_reach(tree=tree, manifest=load_manifest(tree, "evals/repo-eval.yaml"), profile=load_profile("codex"), + model="m", out_dir=tmp_path / "out", kinds=("root", "bash")) + assert not (tmp_path / "out").exists() + + +def test_the_dry_run_plans_the_bash_scenarios(tmp_path, capsys): + root, profile = _setup(tmp_path) + args = build_parser().parse_args(["repo", "reach", str(root), "--manifest", "evals/repo-eval.yaml", "--harness", + str(profile), "--model", "m", "--scenarios", "bash", "--only", "bash=sub", + "--dry-run"]) + args.func(args) + runs = json.loads(capsys.readouterr().out)["runs"] + assert [(r["scenario"], r["cwd"], r["argv"][-2:]) for r in runs] == [("bash=sub", ".", ["--tools", "Bash"])] + assert "run `cat sub/REACH_PROBE.txt`" in runs[0]["argv"][runs[0]["argv"].index("-p") + 1] + + +@pytest.mark.parametrize("mode, cls", [("fetch", "fetched"), ("lazy", "loaded"), ("none", "not_loaded")]) +def test_a_bash_read_of_the_probe_shows_whether_the_directory_guidance_loads(tmp_path, monkeypatch, mode, cls): + payload = _probe(tmp_path, monkeypatch, mode) + assert [s["id"] for s in payload["scenarios"]] == ["bash=sub"] and not payload["scenarios"][0]["error"] + assert payload["summary"]["matrix"] == {"AGENTS.md": {"bash=sub": "loaded"}, "sub/AGENTS.md": {"bash=sub": cls}} + events = [json.loads(x) for x in (tmp_path / "out/traces/00-bash_sub.events.jsonl").read_text().splitlines()] + assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0} + + +def test_a_bash_scenario_receipt_expects_the_lazy_file_and_sees_the_shell_read(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + row = {"id": "bash=sub", "cwd": "", "tools": "bash", "read": "sub/REACH_PROBE.txt", "error": None, "prose": 0, + "unknown": []} + records = [_record("sub/AGENTS.md", "bash=sub", "fetched", read_by_tool=True), + _record("CLAUDE.md", "bash=sub", "not_loaded")] + receipt = build_receipt(_saved_reach(records, [row]), BUDGET, tree) + src = _sources(receipt, "bash=sub") + assert src["sub/AGENTS.md"]["expected"] is True # lazy: the run reads inside sub, as read=sub does + assert src["sub/AGENTS.md"]["states"]["accessed"] is True # the cat was a read event + assert src["CLAUDE.md"]["states"]["accessed"] == "unknown" # a tools-on run: no read is not proof of none + assert next(r for r in receipt["receipts"] if r["run_id"].endswith("bash=sub"))["target"] == "sub/REACH_PROBE.txt" From 862dc498843804691b713ca604b4b0928229330b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:04:44 +0000 Subject: [PATCH 02/42] fix(repo): a cd in a job or pipeline does not move the Bash shell Review of #369 (comments 4443237154, 4443248498): shell_reads_moved() returned the walker's chain flag `moved`, which says a cd was seen, not that the shell the script runs in may have moved. `cd sub &`, `{ cd sub; } | cat`, `{ cd sub; } &` and `cd sub && make &` all came back as moved, so a later Bash call's `cat AGENTS.md` of the same agent lost its root read. _Walk now keeps `shell_moved` apart from `moved`: a cd, pushd or popd sets it only outside a subshell, and it is saved with the directory at the start of an && / || list and at a brace group's opening, and restored wherever the directory is (a backgrounded list, a piped or backgrounded group, and a list backgrounded at the end of the script). `moved` and the reads of Codex and Copilot commands are unchanged. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 7 ++-- dokimi/dokimi/repos/cli_traces.py | 41 ++++++++++--------- dokimi/dokimi/repos/events.py | 3 +- dokimi/tests/test_repo_events_bash.py | 16 ++++++++ 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 43af0d65c..0d737a697 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd`, that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 8df24ef2a..71f5c8d28 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -466,7 +466,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd` (the shell keeps its directory between calls, so the file is not known); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1075,8 +1075,9 @@ per scenario). such read of one file and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative - read after an earlier Bash call of the same agent ran a `cd`, so such a file - can be classed `loaded`; + read after an earlier Bash call of the same agent ran a `cd` in its own + shell (not in a subshell, pipeline or background job), so such a file can be + classed `loaded`; - `not_loaded`; - `error`: the scenario produced no usable answer: no answer, a non-zero exit, or an error result. diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index b9a7a32d4..bccefd6c1 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -101,9 +101,10 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: """:func:`shell_reads`, and whether the script may leave its shell in another directory: it ran a ``cd``, - ``pushd`` or ``popd`` outside a subshell, or a ``cd`` whose target is not known.""" + ``pushd`` or ``popd`` in that shell, not in a subshell (``( ... )``, a pipeline or a backgrounded command, list + or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``).""" reads, walk = _walk_script(script, cwd) - return reads, walk.moved + return reads, walk.shell_moved def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: @@ -117,7 +118,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: op, previous = previous, after chained = op in ("&&", "||") if not chained and walk.gated: # a cd after && or || may not have run: the next chain starts in no known directory - walk.lose() + walk.lose(False) walk.gated = False walk.chain(op, chained) if walk.in_function(_bare(part)): # a definition runs nothing @@ -126,26 +127,28 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: walk.here = None if in_subshell: # a subshell: its reads are not tracked if _runs_dir_before_parens(part): # ``cd $(dirname f)``: the cd is outside the parentheses - walk.lose() + walk.lose(after != "&") continue walk.track_groups(_bare(part), after, out) words = walk.words(part) if words: out.extend(walk.command(words, after, chained)) + walk.chain(previous, False) # a list backgrounded at the end (``cd sub && make &``) ran in a subshell too return out, walk class _Walk: - """Where ``shell_reads`` is in the script: the directory (``None``: unknown), whether a ``cd`` was seen, and - the open brace groups as (directory before it, in a pipeline).""" + """Where ``shell_reads`` is in the script: the directory (``None``: unknown), whether a ``cd`` was seen, whether + the script's own shell may have moved, and the open brace groups.""" def __init__(self, here: str | None) -> None: self.here = here self.moved = False # a cd was seen: after ||, the directory is not known + self.shell_moved = False # a directory builtin ran in the script's shell, not in a subshell self.gated = False # a cd after && or || was seen in this chain: it may not have run for the next chain - # the open brace groups: (directory before it, in a pipeline, index in the reads where it began) - self.groups: list[tuple[str | None, bool, int]] = [] - self.list_start: str | None = here # the directory where the current && / || list began + # the open brace groups: ((here, shell_moved) before it, in a pipeline, index in the reads where it began) + self.groups: list[tuple[tuple[str | None, bool], bool, int]] = [] + self.list_start: tuple[str | None, bool] = (here, False) # (here, shell_moved) where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition @@ -153,11 +156,11 @@ def chain(self, op: str, chained: bool) -> None: """A list of more than one command ended by ``&`` ran in a subshell, so this shell did not move: ``cd sub && cat f & cat g`` reads sub/f and g (a lone ``cd x &`` and a ``{ ...; } &`` group are handled where they end).""" if op == "&" and self.listed: - self.here = self.list_start + self.here, self.shell_moved = self.list_start if chained: self.listed = True else: - self.list_start, self.listed = self.here, False + self.list_start, self.listed = (self.here, self.shell_moved), False def in_function(self, bare: str) -> bool: """Whether ``bare`` is part of a function definition (``f() { cd x; cat a; }``): its body is skipped, as @@ -167,8 +170,10 @@ def in_function(self, bare: str) -> bool: self.function = max(self.function + len(_GROUP_OPEN.findall(bare)) - len(_GROUP_CLOSE.findall(bare)), 0) return True - def lose(self) -> None: + def lose(self, shell: bool = True) -> None: + """The directory is no longer known; ``shell``: because this shell may have moved (not a subshell's cd).""" self.here, self.moved = None, True + self.shell_moved |= shell def track_groups(self, bare: str, after: str, out: list[str]) -> None: """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. The @@ -177,7 +182,7 @@ def track_groups(self, bare: str, after: str, out: list[str]) -> None: opened = _GROUP_OPEN.search(bare) if opened: piped = bool(_PIPE.search(bare[:opened.start()])) - self.groups.append((self.here, piped, len(out))) + self.groups.append(((self.here, self.shell_moved), piped, len(out))) if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked self.here = None closed = _GROUP_CLOSE.search(bare) @@ -187,10 +192,8 @@ def track_groups(self, bare: str, after: str, out: list[str]) -> None: redirects = bare[closed.end():] if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): del out[mark:] - if piped or _PIPE.search(after_close): # the group ran in a pipeline's subshell - self.here = before - elif after == "&": # the group ran in a subshell - self.here = before + if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell + self.here, self.shell_moved = before def words(self, part: str) -> list[str]: """The words of ``part``; none when it cannot be split or is computed (a computed ``cd`` loses the directory).""" @@ -224,12 +227,12 @@ def command(self, words: list[str], after: str, chained: bool = False) -> list[s if name is None: return [] if name in _DIR_STACK or (conditional and name == "cd"): - self.lose() + self.lose(after != "&") elif name == "cd": self.moved = True self.gated |= chained if after != "&": # a backgrounded cd runs in a subshell: this shell stays where it was - self.here = _cd(self.here, words[1:]) + self.here, self.shell_moved = _cd(self.here, words[1:]), True elif name in _PRINTERS: if _asks_for_help(words[1:]): # ``cat --help AGENTS.md`` prints the help and opens no file return [] diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 8bae9b747..dcf2a8865 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -21,7 +21,8 @@ A shell command that only prints files (:func:`~dokimi.repos.cli_traces.shell_reads`) also gives a ``read`` event per file, before its ``command`` event: a Codex or Copilot shell command, and Claude Code's ``Bash`` tool. Claude Code keeps the shell's directory from one ``Bash`` call to the -next, so once a call of an agent ran a ``cd``, that agent's later relative reads are not recorded. +next, so once a call of an agent ran a ``cd`` in that shell (not in a subshell, a pipeline or a +background job), that agent's later relative reads are not recorded. Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` (parent, subagents and helper models) when the result has it, else the parent's diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 3834864c9..eda6eb0b1 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -9,6 +9,7 @@ import pytest +from dokimi.repos.cli_traces import shell_reads_moved from dokimi.repos.events import convert from dokimi.repos.process_metrics import process_metrics @@ -94,6 +95,21 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): assert _reads(events) == ["AGENTS.md"] +@pytest.mark.parametrize("command, moves", [ + ("cd sub &", False), ("{ cd sub; } | cat", False), ("{ cd sub; } &", False), ("cd sub && make &", False), + ("(cd sub)", False), ("cd sub", True), ("cd sub && make", True), ('cd "$X"', True), ("pushd sub", True), + ("popd", True), ("cd sub & cd docs", True), ("{ cd sub; } & cd docs", True), +]) +def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): + assert shell_reads_moved(command)[1] is moves + + +@pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &"]) +def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): + events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) + assert _reads(events) == ["AGENTS.md"] + + def test_a_pushd_in_an_earlier_call_also_loses_the_directory(): events = _events(_call("a", "pushd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == [] From 8f2b8de3af3f6efdab5d244f5a247040f8fcb23f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:07:11 +0000 Subject: [PATCH 03/42] fix(repo): settle Claude Code Bash reads once the trace ends Review of #369 (comments 4443237165, 4443237182). Two orders of the stream-json trace gave wrong reads: - A permission refusal can come back as an `is_error` tool_result (`Permission to use Bash has been denied`) and only later as a `permission_denials` entry in the final result. The error result dropped the read of a one-file `cat AGENTS.md`, so the denial marked a detached event: the trace lost the denied read and `tool_calls` was one short. - A call's `cd` marked the agent's shell as moved at tool_use time, so a `cd sub` that a PreToolUse hook or the permissions refused still cost the next call's `cat AGENTS.md` its read. The converter now records each Bash call (its agent, order, named paths, whether it moves the shell, and ok / error / denied) and settles them all after the last line: a denied call moves nothing and keeps its reads, denied; a failed one-file read drops its read; after a call that was not denied and may have moved the shell, only absolute reads are kept; otherwise reads get rc 0. Read events are still inserted before their command event at tool_use time, and only the dropped ones are removed at the end, so every read keeps its place. events._apply stays at complexity 14. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 6 +- dokimi/dokimi/repos/events.py | 85 ++++++++++++------- dokimi/tests/test_repo_events_bash.py | 24 ++++++ 4 files changed, 83 insertions(+), 34 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 0d737a697..1c7a7b269 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 71f5c8d28..35649ff00 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -466,7 +466,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1076,8 +1076,8 @@ per scenario). `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative read after an earlier Bash call of the same agent ran a `cd` in its own - shell (not in a subshell, pipeline or background job), so such a file can be - classed `loaded`; + shell (not in a subshell, pipeline or background job, and not refused), so + such a file can be classed `loaded`; - `not_loaded`; - `error`: the scenario produced no usable answer: no answer, a non-zero exit, or an error result. diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index dcf2a8865..c6fb475ae 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -22,7 +22,9 @@ ``read`` event per file, before its ``command`` event: a Codex or Copilot shell command, and Claude Code's ``Bash`` tool. Claude Code keeps the shell's directory from one ``Bash`` call to the next, so once a call of an agent ran a ``cd`` in that shell (not in a subshell, a pipeline or a -background job), that agent's later relative reads are not recorded. +background job), that agent's later relative reads are not recorded. A refused call ran nothing: +it moves no shell, and its reads stay, ``denied``. The reads are settled when the trace ends, +since a ``permission_denials`` entry comes only in the final result. Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` (parent, subagents and helper models) when the result has it, else the parent's @@ -34,6 +36,7 @@ import json import re from dataclasses import dataclass, field +from collections.abc import Iterable from datetime import datetime from typing import Any @@ -66,6 +69,18 @@ def convert(fmt: str, raw: str, workdir: str | None = None, cwd: str = "") -> Tr raise ValueError(f"unknown trace format {fmt!r} (known: {', '.join(TRACE_FORMATS)})") +@dataclass +class _BashCall: + """A Bash call, settled when the trace ends: what came back for it decides its reads.""" + + agent: str # "" the parent + text: str + named: list[str] # the files its command prints, from the agent's start directory + reads: list[dict[str, Any]] # their read events, in place before the command event + moves: bool # the command may leave its shell in another directory + result: str | None = None # "ok", "error" or "denied"; None while nothing came back + + @dataclass class _Stream: """What a Claude stream-json trace carries from line to line.""" @@ -73,9 +88,7 @@ class _Stream: workdir: str | None cwd: str # where each agent's shell starts, relative to workdir by_id: dict[str, dict[str, Any]] = field(default_factory=dict) # tool_use id -> its event - # tool_use id of a Bash call -> the read events derived from it, and the files its command names - shell_reads: dict[str, tuple[list[dict[str, Any]], list[str]]] = field(default_factory=dict) - moved: set[str] = field(default_factory=set) # agents ("" the parent) whose shell may have left cwd + bash: dict[int, _BashCall] = field(default_factory=dict) # id() of a Bash call's command event -> the call def _claude_stream_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: @@ -101,6 +114,7 @@ def _claude_stream_json(raw: str, workdir: str | None, cwd: str = "") -> Transcr _apply(out, rec, t, stream) except (AttributeError, TypeError) as exc: # valid JSON of an unexpected shape out.errors.append(f"line {n}: unexpected shape ({exc})") + _settle_bash(out, stream.bash.values()) main = parse_trace(raw).main_calls() out.first_prompt_tokens = main[0]["prompt_tokens"] if main else None return out @@ -118,8 +132,8 @@ def _apply_result(out: Transcript, block: dict[str, Any], stream: _Stream) -> No denied = bool(block.get("is_error")) and bool(_HOOK_DENIAL.match(output or "")) if denied: _deny(stream, tool_id) - else: - _settle_shell_reads(out, stream, tool_id, bool(block.get("is_error"))) + elif id(result_of) in stream.bash: + stream.bash[id(result_of)].result = "error" if block.get("is_error") else "ok" if result_of["kind"] == "read" and not denied: # A read that failed (the file does not exist) shows nothing of the file. result_of["rc"] = 1 if block.get("is_error") else 0 @@ -138,28 +152,42 @@ def _set_num_turns(out: Transcript, turns: Any) -> None: def _deny(stream: _Stream, tool_id: Any) -> None: - """A refused call: its event and the reads derived from it, which never ran.""" - stream.by_id[tool_id]["denied"] = True - for read in stream.shell_reads.get(tool_id, ([], []))[0]: - read["denied"] = True - - -def _settle_shell_reads(out: Transcript, stream: _Stream, tool_id: Any, is_error: bool) -> None: - """The reads of a Bash call that ran: dropped when it failed and was one read of one file (``cat AGENTS.md`` of - a missing file), else ``rc`` 0, since a failure does not say which file failed (``cat a missing``).""" - reads, named = stream.shell_reads.get(tool_id, ([], [])) - if is_error and _failed_read(str(stream.by_id[tool_id].get("text", "")), named, 1): - dropped = {id(e) for e in reads} + """A refused call (a hook's result or a final ``permission_denials`` entry): it never ran.""" + event = stream.by_id[tool_id] + event["denied"] = True + if id(event) in stream.bash: + stream.bash[id(event)].result = "denied" + + +def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: + """Each Bash call's reads, once every result and denial is in. In each agent's order: + + - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``; + - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file) drops its reads; + - after a call that was not denied and may have moved the agent's shell, only absolute reads are known; + - else the reads get ``rc`` 0, since a failure does not say which file failed (``cat a missing``). + """ + moved: set[str] = set() # agents ("" the parent) whose shell may have left its start directory + dropped: set[int] = set() + for call in calls: + failed = call.result == "error" and _failed_read(call.text, call.named, 1) + for read, path in zip(call.reads, call.named, strict=True): + if failed or (call.agent in moved and not path.startswith("/")): + dropped.add(id(read)) + elif call.result == "denied": + read["denied"] = True + elif call.result: + read["rc"] = 0 + if call.moves and call.result != "denied": + moved.add(call.agent) + if dropped: out.events[:] = [e for e in out.events if id(e) not in dropped] - return - for read in reads: - read["rc"] = 0 def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: """The events of one tool_use block: a Bash call's derived reads, then its own event.""" event = _tool_event(block, t, stream.workdir) - reads, named = _bash_reads(event, agent, stream) if block.get("name") == _SHELL_TOOL else ([], []) + reads = _bash_call(event, agent, stream).reads if block.get("name") == _SHELL_TOOL else [] if agent: for e in (*reads, event): e["agent"] = agent @@ -168,17 +196,14 @@ def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: st out.events.extend([*reads, event]) if block.get("id"): stream.by_id[block["id"]] = event - stream.shell_reads[block["id"]] = (reads, named) -def _bash_reads(event: dict[str, Any], agent: str, stream: _Stream) -> tuple[list[dict[str, Any]], list[str]]: - """The read events of a Bash command and the files it names. Once an earlier call of the same agent may have - left the start directory, a relative path names no known file: only absolute ones are kept.""" +def _bash_call(event: dict[str, Any], agent: str, stream: _Stream) -> _BashCall: + """A Bash command's call, with a read event for each file it names; :func:`_settle_bash` keeps or drops them.""" named, moves = shell_reads_moved(str(event["text"]), stream.cwd) - known = [p for p in named if p.startswith("/")] if agent in stream.moved else named - if moves: - stream.moved.add(agent) - return [{"t": event["t"], "kind": "read", "path": relative_path(p, stream.workdir)} for p in known], named + reads = [{"t": event["t"], "kind": "read", "path": relative_path(p, stream.workdir)} for p in named] + stream.bash[id(event)] = call = _BashCall(agent, str(event["text"]), named, reads, moves) + return call def _apply(out: Transcript, rec: dict[str, Any], t: float | None, stream: _Stream) -> None: diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index eda6eb0b1..88e00c7b4 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -70,6 +70,30 @@ def test_a_refused_bash_call_marks_its_reads_denied(refusal): assert process_metrics(events, lambda p: p.endswith("AGENTS.md"))["guidance_read"] == [] +def _denials(*tool_ids: str) -> dict: + return {"type": "result", "result": "ok", + "permission_denials": [{"tool_name": "Bash", "tool_use_id": i} for i in tool_ids]} + + +def test_a_permission_error_then_its_denial_keeps_the_read_denied_in_place(): + events = _events(_call("a", "cat AGENTS.md"), _result("a", "Permission to use Bash has been denied", True), + _call("b", "cat README.md"), _result("b"), _denials("a")) + assert [(e["kind"], e.get("path"), e.get("denied")) for e in events] == [ + ("read", "AGENTS.md", True), ("command", None, True), ("read", "README.md", None), ("command", None, None)] + assert process_metrics(events, lambda p: p.endswith("AGENTS.md"))["tool_calls"] == 4 + + +@pytest.mark.parametrize("refusal, reads", [ + ("hook", ["AGENTS.md"]), ("permissions", ["AGENTS.md"]), (None, []), # None: the cd ran +]) +def test_a_refused_cd_moves_no_shell(refusal, reads): + text = {"hook": "PreToolUse:Bash hook error: no cd", "permissions": "Permission to use Bash has been denied"} + events = _events(_call("a", "cd sub"), _result("a", text.get(refusal, ""), bool(refusal)), + _call("b", "cat AGENTS.md"), _result("b"), _denials("a") if refusal == "permissions" else {}) + assert _reads(events) == reads + assert [e["kind"] for e in events] == ["command", *(["read"] if reads else []), "command"] + + @pytest.mark.parametrize("command", ["cat AGENTS.md | head", "grep -n rule AGENTS.md", "cat AGENTS.md > copy.md"]) def test_a_pipe_a_search_or_a_redirect_is_no_read(command): assert _reads(_events(_call("a", command), _result("a"))) == [] From aa0c86f000edf5d7668b2cda8ecade31db7c67cc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:09:51 +0000 Subject: [PATCH 04/42] fix(repo): a refused or failed read is no read in reach or skill use Review of #369 (comment 4443248578): Transcript.reads(), which reach's classify and so the receipt's `read_by_tool` use, and arms.skill_usage took every read event, including denied ones and failed ones (rc 1). A refused `cat sub/AGENTS.md` made an already-loaded file `fetched` and its receipt `states.accessed` true, and a refused read of an installed SKILL.md counted the skill as used. The Read tool had the same bug before the Bash reads of this PR. Both now take only reads that showed the file (transcript.shown_read: not denied, rc 0 or no result), as process_metrics' guidance_read already did. The events stay in events.jsonl for audit. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 7 +++--- dokimi/dokimi/repos/arms.py | 7 ++++-- dokimi/dokimi/repos/transcript.py | 8 +++++- dokimi/tests/test_repo_reach_bash.py | 25 +++++++++++++++---- dokimi/tests/test_repo_skill_arms.py | 21 ++++++++++++++++ 6 files changed, 58 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 1c7a7b269..6d8b38007 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 35649ff00..baaf84f22 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -474,7 +474,7 @@ parent's and subagents' together, so a grader need not: A cell of an arm that installs skills also has a `skills` object in `result.json`: `measured`, `skills_installed` (the installed skill names), `skills_used` (the installed skills the agent called with the `Skill` tool or -whose `SKILL.md` it read, in order of first use), `expected_skill` (the task's, +whose `SKILL.md` it read, the read neither denied nor failed, in order of first use), `expected_skill` (the task's, or `null`), `skill_invoked` (the expected skill was used; any installed skill when the task names none) and `skill_selected_correct` (the expected skill was the first one used; `null` when the task names none). When the trace could not @@ -1069,7 +1069,8 @@ per scenario). - `class` is one of: - `loaded`: both tokens reported; - `truncated`: one token reported; - - `fetched`: reported only after the agent read the file with a tool. For the + - `fetched`: reported only after the agent read the file with a tool (a + read that was denied or failed is none). For the Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one such read of one file and exits non-zero is not a read (the exit code of @@ -1132,7 +1133,7 @@ Each source carries: | `observed` | reach class | `loaded` → loaded, `fetched` → fetched, `not_loaded` → not_loaded, `truncated` and `error` → unknown | | `observation_method` | | `unknown` for every source: the thread vocabulary's method, since no runtime event, prompt capture or hook backs the probe | | `dokimi_method` | | `marker_self_report` for a source the probe recorded; null for a start-up file budget found and reach did not probe | -| `states.accessed` | the harness's read events | `true` if the trace has a read event of the file (a read tool, or a shell read §7 lists, under all three harnesses); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | +| `states.accessed` | the harness's read events | `true` if the trace has a read event of the file, neither denied nor failed (a read tool, or a shell read §7 lists, under all three harnesses); for a probed source `false` with tools off, else `unknown`; always `unknown` for a start-up file budget found and reach did not probe | | `states.selected` | the class | `true` for loaded or fetched (the source entered the request, by the harness or a read), else `unknown` | | `states.loaded` | the class | `true` for loaded, `false` for not_loaded, `unknown` for fetched, truncated and error: a tool read is not proof that start-up loading did not happen | | `states.followed` | | always `unknown` (the schema's `const`) | diff --git a/dokimi/dokimi/repos/arms.py b/dokimi/dokimi/repos/arms.py index ee96d526c..98f106b39 100644 --- a/dokimi/dokimi/repos/arms.py +++ b/dokimi/dokimi/repos/arms.py @@ -29,6 +29,8 @@ from dokimi import process +from .transcript import shown_read + SKILL_KINDS = ("skill", "skill-catalog") SKILLS_DIR = ".claude/skills" @@ -254,7 +256,8 @@ def skill_unmeasured(installed: list[str], expected: str | None) -> dict[str, An def skill_usage(events: list[dict[str, Any]], installed: list[str], expected: str | None) -> dict[str, Any]: - """Which installed skills the agent used: a ``Skill`` tool call naming it, or a read of its ``SKILL.md``. + """Which installed skills the agent used: a ``Skill`` tool call naming it, or a read of its ``SKILL.md`` that was + neither refused nor failed. ``skill_invoked``: the expected skill was used (any installed one when the task names none); ``skill_selected_correct``: the expected skill was the first one used, ``None`` without one. @@ -264,7 +267,7 @@ def skill_usage(events: list[dict[str, Any]], installed: list[str], expected: st name = None if e.get("kind") == "tool" and e.get("name") == "Skill": name = str(e.get("skill") or "").split(":")[-1] # a plugin skill is plugin:name - elif e.get("kind") == "read": + elif shown_read(e): # a refused or failed read of a SKILL.md showed nothing of the skill parts = PurePosixPath(str(e.get("path") or "")).parts if len(parts) == 4 and "/".join(parts[:2]) == SKILLS_DIR and parts[3] == "SKILL.md": name = parts[2] diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 2411f8e72..8b9efacf4 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -53,12 +53,18 @@ def model_resolved(self) -> str | list[str] | None: return (found[0] if len(found) == 1 else found) if found else None def reads(self) -> set[str]: - return {e["path"] for e in self.events if e["kind"] == "read" and e.get("path")} + """The paths the agent was shown (:func:`shown_read`); a refused or failed read stays in the events only.""" + return {e["path"] for e in self.events if shown_read(e) and e.get("path")} def events_jsonl(self) -> str: return "".join(json.dumps(e, sort_keys=True) + "\n" for e in self.events) +def shown_read(event: dict[str, Any]) -> bool: + """A read event that showed the agent its file: not refused (``denied``), not failed (``rc`` 0, or no result).""" + return event.get("kind") == "read" and not event.get("denied") and event.get("rc") in (0, None) + + def clip_output(text: str) -> tuple[str, bool]: """``text`` kept whole up to :data:`OUTPUT_LIMIT` UTF-8 bytes, else its head and tail.""" data = text.encode("utf-8") diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 340577425..5e8831f86 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -5,7 +5,8 @@ The fake harness prints Claude stream-json. With its Bash tool on and a prompt that names a ``cat``, it runs that ``cat`` as a Bash call; ``FAKE_MODE`` says what follows: ``lazy`` loads the file's directory's AGENTS.md as context, -``fetch`` also cats that AGENTS.md itself, ``none`` loads nothing more. +``fetch`` also cats that AGENTS.md itself, ``none`` loads nothing more; ``hook`` and ``permissions`` load it as +``lazy`` does, after a ``cat`` of it that a hook or the permissions refuse. """ from __future__ import annotations @@ -40,22 +41,26 @@ root = Path.cwd() context = [root / "AGENTS.md"] events = [{"type": "system", "subtype": "init", "claude_code_version": "9.8.7"}] -def bash(n, command, output): +REFUSALS = {"hook": "PreToolUse:Bash hook error: no", "permissions": "Permission to use Bash has been denied"} +def bash(n, command, output, error=False): events.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": f"b{n}", "name": "Bash", "input": {"command": command}}]}}) events.append({"type": "user", "message": {"content": [ - {"type": "tool_result", "tool_use_id": f"b{n}", "is_error": False, "content": output}]}}) + {"type": "tool_result", "tool_use_id": f"b{n}", "is_error": error, "content": output}]}}) match = re.search(r"run `cat (\S+)`", prompt) if match and tools == "Bash": probe = Path(match.group(1)) bash(1, f"cat {probe}", (root / probe).read_text()) if mode == "fetch": bash(2, f"cat {probe.parent}/AGENTS.md", (root / probe.parent / "AGENTS.md").read_text()) - if mode in ("lazy", "fetch"): + if mode in REFUSALS: # the cat of the guidance file is refused; the file was loaded lazily all the same + bash(2, f"cat {probe.parent}/AGENTS.md", REFUSALS[mode], True) + if mode != "none": context.append(root / probe.parent / "AGENTS.md") text = "".join(p.read_text() for p in context) answer = "\n".join(dict.fromkeys(re.findall(r"RC[0-9A-F]{6}", text))) or "NONE" -events.append({"type": "result", "subtype": "success", "is_error": False, "result": answer, +denials = [{"tool_name": "Bash", "tool_use_id": "b2"}] if mode == "permissions" else [] +events.append({"type": "result", "subtype": "success", "is_error": False, "result": answer, "permission_denials": denials, "total_cost_usd": 0.001, "num_turns": 2, "usage": {"input_tokens": 10, "output_tokens": 5}}) for e in events: print(json.dumps(e)) @@ -151,6 +156,16 @@ def test_a_bash_read_of_the_probe_shows_whether_the_directory_guidance_loads(tmp assert events[0] == {"t": None, "kind": "read", "path": "sub/REACH_PROBE.txt", "rc": 0} +@pytest.mark.parametrize("mode", ["hook", "permissions"]) +def test_a_refused_bash_read_of_the_guidance_is_no_fetch(tmp_path, monkeypatch, mode): + payload = _probe(tmp_path, monkeypatch, mode) + assert payload["summary"]["matrix"]["sub/AGENTS.md"] == {"bash=sub": "loaded"} + events = [json.loads(x) for x in (tmp_path / "out/traces/00-bash_sub.events.jsonl").read_text().splitlines()] + assert [e.get("denied") for e in events if e.get("path") == "sub/AGENTS.md"] == [True] # kept for audit + receipt = build_receipt(payload, BUDGET, WorkTree(tmp_path / "repo")) + assert _sources(receipt, "bash=sub")["sub/AGENTS.md"]["states"]["accessed"] == "unknown" + + def test_a_bash_scenario_receipt_expects_the_lazy_file_and_sees_the_shell_read(tmp_path): tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) row = {"id": "bash=sub", "cwd": "", "tools": "bash", "read": "sub/REACH_PROBE.txt", "error": None, "prose": 0, diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py index c099ec173..6a523b4ce 100644 --- a/dokimi/tests/test_repo_skill_arms.py +++ b/dokimi/tests/test_repo_skill_arms.py @@ -94,6 +94,27 @@ def test_skill_usage_from_skill_calls_and_skill_md_reads(): assert plugin["skill_invoked"] is True and plugin["skill_selected_correct"] is True +@pytest.mark.parametrize("result, final", [ + ("PreToolUse:Bash hook error: no", {}), # a hook refused it + ("Permission to use Bash has been denied", {"permission_denials": [{"tool_name": "Bash", "tool_use_id": "a"}]}), +]) +def test_a_refused_read_of_skill_md_is_no_use_of_the_skill(result, final): + lines = [{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "a", "name": "Bash", + "input": {"command": "cat .claude/skills/lint/SKILL.md"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "a", "is_error": True, + "content": result}]}}, + {"type": "result", "result": "ok", **final}] + events = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws").events + assert [e.get("denied") for e in events if e["kind"] == "read"] == [True] # kept in the trace, for audit + assert skill_usage(events, ["lint"], "lint")["skills_used"] == [] + + +def test_a_failed_read_of_skill_md_is_no_use_of_the_skill(): + failed = {"kind": "read", "path": ".claude/skills/lint/SKILL.md", "rc": 1} + assert skill_usage([failed], ["lint"], "lint")["skills_used"] == [] + assert skill_usage([{**failed, "rc": 0}], ["lint"], "lint")["skills_used"] == ["lint"] + + def test_only_a_read_of_skill_md_counts_as_using_a_skill(): for path in (".claude/skills/lint/references/x.md", ".claude/skills/lint/notes.md"): used = skill_usage([{"kind": "read", "path": path}], ["lint"], "lint") From d0bb1ab144bfec1d8b88b5e91a7fd0a7db500fdd Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:11:20 +0000 Subject: [PATCH 05/42] fix(repo): shell-quote the probe path in the bash= reach prompt Review of #369 (comments 4443237195, 4443248319): the bash= prompt put the probe path into the Bash command as it was. Repository directory names keep spaces and shell metacharacters, so guidance in `my package/AGENTS.md` asked for `cat my package/REACH_PROBE.txt`, which reads `my` and `package/REACH_PROBE.txt`, and a `$` or `;` in the name changed what the command does. The operand is now reach.shell_word(path): shlex.quote of the path, with `./` in front of a path that starts with `-`. Scenario.read keeps the raw path, and the Read-tool prompt is unchanged. The fake harness of test_repo_reach_bash.py parses the quoted operand with shlex.split and runs the command as the prompt names it. Signed-off-by: Anton Volkov --- ...2_120100_antonwolfy_reach-bash-scenario.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 +- dokimi/dokimi/repos/reach.py | 11 +++++- dokimi/tests/test_repo_reach_bash.py | 37 ++++++++++++++----- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md index ecec5a37b..5ea380c2c 100644 --- a/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md +++ b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **A `bash=` reach scenario** — `repo reach --scenarios bash` runs, for each guidance directory, a scenario started at the root with the profile's new `bash` tool mode (`claude-code`: `--tools Bash`) that tells the agent to `cat /REACH_PROBE.txt` and then list tokens: it shows whether a shell read of a nested file loads that directory's guidance, as `read=` shows for the Read tool. The default stays `root,cwd,read`, so a probe costs no more unless asked; `--dry-run` lists the bash scenarios and `--only bash=` picks one. A profile without a `bash` tool mode (`codex`, `copilot`) asked for `--scenarios bash` is a usage error (exit 2) naming the profile. `harness_profile.v1` accepts an optional `bash` key under `tools` (not under `delegation.none`, so `repo run --tools bash --delegation none` is refused); the receipt treats a `bash=` run as it does `read=`. +- **A `bash=` reach scenario** — `repo reach --scenarios bash` runs, for each guidance directory, a scenario started at the root with the profile's new `bash` tool mode (`claude-code`: `--tools Bash`) that tells the agent to `cat /REACH_PROBE.txt` (the path shell-quoted, so a space or `;` in a directory name stays one operand) and then list tokens: it shows whether a shell read of a nested file loads that directory's guidance, as `read=` shows for the Read tool. The default stays `root,cwd,read`, so a probe costs no more unless asked; `--dry-run` lists the bash scenarios and `--only bash=` picks one. A profile without a `bash` tool mode (`codex`, `copilot`) asked for `--scenarios bash` is a usage error (exit 2) naming the profile. `harness_profile.v1` accepts an optional `bash` key under `tools` (not under `delegation.none`, so `repo run --tools bash --delegation none` is refused); the receipt treats a `bash=` run as it does `read=`. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index baaf84f22..769d0be8f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1033,7 +1033,8 @@ Protocol: read `/REACH_PROBE.txt`, then list tokens. - `bash=`: started at the root, the profile's `bash` tool mode (Claude Code: the Bash tool alone). The agent is told to run - `cat /REACH_PROBE.txt`, then list tokens: it shows whether a shell + `cat /REACH_PROBE.txt` (the path shell-quoted, `./` before a leading + `-`), then list tokens: it shows whether a shell read of a file in `` loads that directory's guidance. Only with `--scenarios bash` (the default is `root,cwd,read`); a profile without a `bash` tool mode (Codex and Copilot, whose shell commands are already their reads) is a diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 13cd8f026..49f9a1e45 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -40,6 +40,7 @@ import hashlib import os import re +import shlex import shutil import tempfile from dataclasses import dataclass @@ -112,7 +113,15 @@ class Scenario: def prompt(self) -> str: if not self.read: return ASK - return (ASK_AFTER_BASH if self.tools == "bash" else ASK_AFTER_READ).format(path=self.read) + ASK + if self.tools == "bash": + return ASK_AFTER_BASH.format(path=shell_word(self.read)) + ASK + return ASK_AFTER_READ.format(path=self.read) + ASK + + +def shell_word(path: str) -> str: + """``path`` as one operand of a POSIX command: quoted (a space, ``$`` or ``;`` in a directory name), and with + ``./`` before a leading ``-``, which would make it an option.""" + return shlex.quote(f"./{path}" if path.startswith("-") else path) def token(salt: str, path: str, position: str) -> str: diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 5e8831f86..156b77b87 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import re import sys from pathlib import Path @@ -20,15 +21,16 @@ from dokimi.cli import build_parser from dokimi.repos import WorkTree, load_manifest +from dokimi.repos.cli_traces import shell_reads from dokimi.repos.harness_profile import ProfileError, load_profile -from dokimi.repos.reach import ASK, ASK_AFTER_BASH, ReachError, run_reach, scenarios +from dokimi.repos.reach import ASK, ASK_AFTER_BASH, ASK_AFTER_READ, ReachError, run_reach, scenarios from dokimi.repos.receipt import build_receipt from tests.test_repo_reach import _write from tests.test_repo_receipt import BUDGET, _record, _sources, _tree from tests.test_repo_receipt import _reach as _saved_reach BASH_HARNESS = r''' -import json, os, re, sys +import json, os, re, shlex, sys from pathlib import Path args = sys.argv[1:] @@ -47,10 +49,10 @@ def bash(n, command, output, error=False): {"type": "tool_use", "id": f"b{n}", "name": "Bash", "input": {"command": command}}]}}) events.append({"type": "user", "message": {"content": [ {"type": "tool_result", "tool_use_id": f"b{n}", "is_error": error, "content": output}]}}) -match = re.search(r"run `cat (\S+)`", prompt) +match = re.search(r"run `cat ([^`]+)`", prompt) if match and tools == "Bash": - probe = Path(match.group(1)) - bash(1, f"cat {probe}", (root / probe).read_text()) + probe = Path(shlex.split(match.group(1))[0]) + bash(1, f"cat {match.group(1)}", (root / probe).read_text()) if mode == "fetch": bash(2, f"cat {probe.parent}/AGENTS.md", (root / probe.parent / "AGENTS.md").read_text()) if mode in REFUSALS: # the cat of the guidance file is refused; the file was loaded lazily all the same @@ -67,11 +69,11 @@ def bash(n, command, output, error=False): ''' -def _setup(tmp_path: Path) -> tuple[Path, Path]: +def _setup(tmp_path: Path, sub: str = "sub") -> tuple[Path, Path]: """A repository with a root and a nested AGENTS.md, and a fake profile with a ``bash`` tool mode.""" root = tmp_path / "repo" _write(root, "AGENTS.md", "# root\n") - _write(root, "sub/AGENTS.md", "# sub\n") + _write(root, f"{sub}/AGENTS.md", "# sub\n") _write(root, "evals/tasks/.keep", "") _write(root, "evals/repo-eval.yaml", yaml.safe_dump({ "schema": "repo_manifest.v1", "id": "toy", "eval_dir": "evals", "guidance": {"globs": ["**/AGENTS.md"]}, @@ -89,9 +91,9 @@ def _setup(tmp_path: Path) -> tuple[Path, Path]: return root, profile -def _probe(tmp_path: Path, monkeypatch, mode: str) -> dict: +def _probe(tmp_path: Path, monkeypatch, mode: str, sub: str = "sub") -> dict: monkeypatch.setenv("FAKE_MODE", mode) - root, profile = _setup(tmp_path) + root, profile = _setup(tmp_path, sub) tree = WorkTree(root) (tmp_path / "work").mkdir() return run_reach(tree=tree, manifest=load_manifest(tree, "evals/repo-eval.yaml"), profile=load_profile(profile), @@ -109,6 +111,23 @@ def test_bash_scenarios_cat_each_probe_file_from_the_root(): assert not [s for s in scenarios(files) if s.tools == "bash"] # the default kinds run no bash scenario +@pytest.mark.parametrize("d", ["my package", "a$b", "a;b", "-x"]) +def test_the_bash_prompt_names_the_probe_as_one_shell_operand(d): + got = scenarios([f"{d}/AGENTS.md"], ("bash",))[0] + command = re.search(r"run `([^`]+)`", got.prompt).group(1) + assert shell_reads(command) == [f"{d}/REACH_PROBE.txt"] + assert got.read == f"{d}/REACH_PROBE.txt" # the raw path, which the Read-tool prompt names as it is + read = scenarios([f"{d}/AGENTS.md"], ("read",))[0] + assert read.prompt == ASK_AFTER_READ.format(path=f"{d}/REACH_PROBE.txt") + ASK + + +def test_a_fake_bash_run_reads_the_probe_of_a_directory_with_a_space(tmp_path, monkeypatch): + payload = _probe(tmp_path, monkeypatch, "lazy", sub="my docs") + assert payload["summary"]["matrix"]["my docs/AGENTS.md"] == {"bash=my docs": "loaded"} + trace = next((tmp_path / "out/traces").glob("*.events.jsonl")) + assert json.loads(trace.read_text().splitlines()[0])["path"] == "my docs/REACH_PROBE.txt" + + def test_the_claude_code_profile_runs_the_bash_mode_with_the_bash_tool_alone(): profile = load_profile("claude-code") assert profile.argv(prompt="p", model="m", tools="bash")[-2:] == ["--tools", "Bash"] From 261b3109afd4d94a7998518bf5e679f3c3f45e1f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:11:44 +0000 Subject: [PATCH 06/42] test(repo): POSIX paths in the fake Bash commands of the bash= tests Review of #369 (comment 4443248422): the fake harness of test_repo_reach_bash.py built its Bash commands from str(Path). On Windows that gives `cat sub\AGENTS.md`, which the POSIX shell parser reads as `subAGENTS.md`, so the read-event and fetched assertions would fail there. The probe command is the prompt's own operand (since the quoting fix), and the command that cats the directory's AGENTS.md is built with as_posix() and shell-quoted. Test-only change. Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_reach_bash.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index 156b77b87..dd7c9f6d9 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -53,10 +53,11 @@ def bash(n, command, output, error=False): if match and tools == "Bash": probe = Path(shlex.split(match.group(1))[0]) bash(1, f"cat {match.group(1)}", (root / probe).read_text()) + guidance = shlex.quote((probe.parent / "AGENTS.md").as_posix()) # not str(): a Windows path has backslashes if mode == "fetch": - bash(2, f"cat {probe.parent}/AGENTS.md", (root / probe.parent / "AGENTS.md").read_text()) + bash(2, f"cat {guidance}", (root / probe.parent / "AGENTS.md").read_text()) if mode in REFUSALS: # the cat of the guidance file is refused; the file was loaded lazily all the same - bash(2, f"cat {probe.parent}/AGENTS.md", REFUSALS[mode], True) + bash(2, f"cat {guidance}", REFUSALS[mode], True) if mode != "none": context.append(root / probe.parent / "AGENTS.md") text = "".join(p.read_text() for p in context) From 8be264e66d745baffc97d346cae014c9cd5270ee Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:51:36 +0000 Subject: [PATCH 07/42] fix(repo): a brace group's separators do not end the list around it `cd sub && { echo a; echo b; } &` backgrounds the whole && list, so the shell does not move, yet shell_reads_moved returned moved=True: the `;` inside the group overwrote the saved start of the enclosing list (`list_start` / `listed`), and the final `&` restored the state after `cd sub` instead of the state before it. A following Claude Code Bash `cat AGENTS.md` then lost its root read. A brace group now saves the enclosing list's state when it opens and restores it when it closes, so the group is one command of that list. `cd sub && { cd x; cat a; } &` moves nothing either, while `cd sub && { echo a; }` (not backgrounded) still moves. The same fix makes `cd x && { cat f; cat h; } & cat g` read `g`, not `x/g`. Settles review comment 4443584029 on #369. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/cli_traces.py | 15 ++++++++------- dokimi/tests/test_repo_cli_traces.py | 1 + dokimi/tests/test_repo_events_bash.py | 5 ++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index bccefd6c1..e3af170b6 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -146,8 +146,9 @@ def __init__(self, here: str | None) -> None: self.moved = False # a cd was seen: after ||, the directory is not known self.shell_moved = False # a directory builtin ran in the script's shell, not in a subshell self.gated = False # a cd after && or || was seen in this chain: it may not have run for the next chain - # the open brace groups: ((here, shell_moved) before it, in a pipeline, index in the reads where it began) - self.groups: list[tuple[tuple[str | None, bool], bool, int]] = [] + # the open brace groups: ((here, shell_moved) before it, in a pipeline, index in the reads where it began, + # the enclosing && / || list's (list_start, listed), which the group's own separators do not end) + self.groups: list[tuple[tuple[str | None, bool], bool, int, tuple[tuple[str | None, bool], bool]]] = [] self.list_start: tuple[str | None, bool] = (here, False) # (here, shell_moved) where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition @@ -176,18 +177,18 @@ def lose(self, shell: bool = True) -> None: self.shell_moved |= shell def track_groups(self, bare: str, after: str, out: list[str]) -> None: - """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. The - reads of a group whose output feeds a pipe after it, or a file, are dropped from ``out``: its output is not - the agent's.""" + """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. A group is + one command of the list around it: ``cd sub && { a; b; } &`` backgrounds the whole list. The reads of a group + whose output feeds a pipe after it, or a file, are dropped from ``out``: its output is not the agent's.""" opened = _GROUP_OPEN.search(bare) if opened: piped = bool(_PIPE.search(bare[:opened.start()])) - self.groups.append(((self.here, self.shell_moved), piped, len(out))) + self.groups.append(((self.here, self.shell_moved), piped, len(out), (self.list_start, self.listed))) if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked self.here = None closed = _GROUP_CLOSE.search(bare) if closed and self.groups: - before, piped, mark = self.groups.pop() + before, piped, mark, (self.list_start, self.listed) = self.groups.pop() after_close = bare[closed.start():] redirects = bare[closed.end():] if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 50f20685e..e0fe36c97 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -76,6 +76,7 @@ ("{ cd x; } & cat f", ["f"]), # a backgrounded group runs in a subshell ("{ cd x; } 2>&1 | cat; cat f", ["f"]), # so does a piped one ("{ cd x; cat f; } & cat g", ["x/f", "g"]), + ("cd x && { cat f; cat h; } & cat g", ["x/f", "x/h", "g"]), # the group is one command of the backgrounded list ("cat a | { cd x; cat b; }", []), # a group in a pipeline: directory unknown after it ("if true; then cd sub; fi; cat AGENTS.md", []), # a conditional cd may not run ("for d in sub; do cd sub; done; cat AGENTS.md", []), diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 88e00c7b4..1e2964dee 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -123,12 +123,15 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("cd sub &", False), ("{ cd sub; } | cat", False), ("{ cd sub; } &", False), ("cd sub && make &", False), ("(cd sub)", False), ("cd sub", True), ("cd sub && make", True), ('cd "$X"', True), ("pushd sub", True), ("popd", True), ("cd sub & cd docs", True), ("{ cd sub; } & cd docs", True), + ("cd sub && { echo a; echo b; } &", False), ("cd sub && { cd x; cat a; } &", False), + ("cd sub && { echo a; }", True), ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves -@pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &"]) +@pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &", + "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &"]) def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == ["AGENTS.md"] From 444aec9d8357cb95a64c4c2cebec68c3423a2e4d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 14:52:32 +0000 Subject: [PATCH 08/42] fix(repo): a computed cd in a job or a pipeline moves no Bash shell `cd "$X" &` and `cd sub | cat` run their `cd` in a subshell, yet shell_reads_moved returned moved=True: a computed part (an expansion, a pipe or a redirect) is not split into words, and `_Walk.words` called `lose()` for its `cd` without knowing the part was backgrounded or piped. A following Claude Code Bash `cat AGENTS.md` then lost its read. `words` now takes the part's operator after it; for a backgrounded part or a pipeline (an unquoted, unescaped `|`), `lose()` still drops the directory for the rest of the script but does not mark the shell moved. `cd "$X" &`, `cd "$X" | cat`, `cd sub | cat` and `echo | cd sub` move nothing; `cd "$X"`, `cd "$X" && make`, `cd $(pwd)` and `cd a\|b` still move. Settles review comment 4443584037 on #369. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/cli_traces.py | 11 ++++++----- dokimi/tests/test_repo_events_bash.py | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index e3af170b6..faef86db2 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -102,7 +102,7 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: """:func:`shell_reads`, and whether the script may leave its shell in another directory: it ran a ``cd``, ``pushd`` or ``popd`` in that shell, not in a subshell (``( ... )``, a pipeline or a backgrounded command, list - or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``).""" + or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``, ``cd "$X" &``, ``cd sub | cat``).""" reads, walk = _walk_script(script, cwd) return reads, walk.shell_moved @@ -130,7 +130,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: walk.lose(after != "&") continue walk.track_groups(_bare(part), after, out) - words = walk.words(part) + words = walk.words(part, after) if words: out.extend(walk.command(words, after, chained)) walk.chain(previous, False) # a list backgrounded at the end (``cd sub && make &``) ran in a subshell too @@ -196,8 +196,9 @@ def track_groups(self, bare: str, after: str, out: list[str]) -> None: if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell self.here, self.shell_moved = before - def words(self, part: str) -> list[str]: - """The words of ``part``; none when it cannot be split or is computed (a computed ``cd`` loses the directory).""" + def words(self, part: str, after: str = "") -> list[str]: + """The words of ``part``; none when it cannot be split or is computed (a computed ``cd`` loses the directory, + and moves this shell unless ``part`` is a pipeline or is backgrounded by ``after``: then it ran in a subshell).""" if not _STDOUT_AFTER_STDERR.search(_bare(part)): # else ``>&2`` stays: a redirect, so no read part = _outside_quotes(part, _HARMLESS_REDIRECT) part = _outside_quotes(part, _STDERR_FILE) @@ -211,7 +212,7 @@ def words(self, part: str) -> list[str]: rough = rough[1:] rough, via_env, _ = _command_words(rough) if rough[:1] and rough[0] in _DIRS and not via_env: # ``builtin cd "$D"``: where it went is not known - self.lose() + self.lose(after != "&" and not _PIPE.search(re.sub(r"\\.", "", _bare(part)))) # ``\|`` is no pipe return [] return words[1:] if words[:1] == ["{"] else words # a brace group runs in this shell: its cd and reads count diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 1e2964dee..4444701a1 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -125,13 +125,16 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("popd", True), ("cd sub & cd docs", True), ("{ cd sub; } & cd docs", True), ("cd sub && { echo a; echo b; } &", False), ("cd sub && { cd x; cat a; } &", False), ("cd sub && { echo a; }", True), + ('cd "$X" &', False), ('cd "$X" | cat', False), ("cd sub | cat", False), ("echo | cd sub", False), + ('cd "$X" && make', True), ("cd $(pwd)", True), (r"cd a\|b", True), ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &", - "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &"]) + "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &", + 'cd "$X" &', 'cd "$X" | cat', "cd sub | cat", "echo | cd sub"]) def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == ["AGENTS.md"] From f05363fadab69b1bd9064681b6dee5f14380feaa Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:00:32 +0000 Subject: [PATCH 09/42] fix(repo): one moved Bash shell for the parent and its subagents The Claude Code converter kept the "shell may have moved" state per agent: after the parent ran `cd sub`, a subagent's `cat AGENTS.md` still gave a read of the root AGENTS.md (and the reverse). Claude Code keeps the Bash directory as process state, and a Task subagent may well share it, so such a read may name a file the agent was never shown. `_settle_bash` now keeps one flag for the whole trace: once any non-denied call may have moved the shell, every agent's later relative reads are dropped, absolute ones kept. This only drops reads, so it is safe whether subagents share the parent's shell or not. The per-agent field of `_BashCall` goes. The test that pinned the per-agent model is replaced by one test per direction (parent cd, then a subagent's read; subagent cd, then the parent's). The contract's guidance_read row, the `fetched` bullet and the "cannot show" list say so. Settles review comment 4445790393 on #369. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 12 +++++--- dokimi/dokimi/repos/events.py | 30 +++++++++---------- dokimi/tests/test_repo_events_bash.py | 23 ++++++++------ 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 6d8b38007..d1b83bc7b 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call of an agent that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), that agent's later relative reads are not recorded (absolute ones are). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 769d0be8f..1604e2d1c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -466,7 +466,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call of the same agent ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1077,9 +1077,10 @@ per scenario). such read of one file and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative - read after an earlier Bash call of the same agent ran a `cd` in its own - shell (not in a subshell, pipeline or background job, and not refused), so - such a file can be classed `loaded`; + read after an earlier Bash call of any agent (the parent's or a + subagent's, which may share its shell) ran a `cd` in its own shell (not in + a subshell, pipeline or background job, and not refused), so such a file + can be classed `loaded`; - `not_loaded`; - `error`: the scenario produced no usable answer: no answer, a non-zero exit, or an error result. @@ -1178,6 +1179,9 @@ observations of that run. It cannot show: - a shell read that the converter does not turn into a read event: under Claude Code, Codex and Copilot alike, only the shell commands §7 lists are read events, so `rg`, `grep` or a pipe stays unseen; +- which directory a Claude Code subagent's Bash shell is in: the converter + assumes it shares the parent's, so a `cd` in either drops the later + relative reads of both (this only drops reads; no real trace was checked); - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: they appear with `observed: unknown`. A skill body is loaded on demand, so it diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index c6fb475ae..6f0f8c5bd 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -21,8 +21,9 @@ A shell command that only prints files (:func:`~dokimi.repos.cli_traces.shell_reads`) also gives a ``read`` event per file, before its ``command`` event: a Codex or Copilot shell command, and Claude Code's ``Bash`` tool. Claude Code keeps the shell's directory from one ``Bash`` call to the -next, so once a call of an agent ran a ``cd`` in that shell (not in a subshell, a pipeline or a -background job), that agent's later relative reads are not recorded. A refused call ran nothing: +next, so once any call ran a ``cd`` in that shell (not in a subshell, a pipeline or a background +job), later relative reads are not recorded, the parent's and every subagent's alike: subagents may +share the parent's shell, and dropping a read is the safe side either way. A refused call ran nothing: it moves no shell, and its reads stay, ``denied``. The reads are settled when the trace ends, since a ``permission_denials`` entry comes only in the final result. @@ -59,7 +60,7 @@ def convert(fmt: str, raw: str, workdir: str | None = None, cwd: str = "") -> Tr """Convert a raw harness stdout in format ``fmt`` to a :class:`Transcript`. ``cwd`` is the directory the harness started in, relative to ``workdir``: a shell command's relative - paths are read from there (in Claude Code, every agent's first ``Bash`` call starts there).""" + paths are read from there (in Claude Code, the first ``Bash`` call starts there).""" if fmt == "text": return Transcript(answer=raw.strip()) if fmt == "claude-stream-json": @@ -73,9 +74,8 @@ def convert(fmt: str, raw: str, workdir: str | None = None, cwd: str = "") -> Tr class _BashCall: """A Bash call, settled when the trace ends: what came back for it decides its reads.""" - agent: str # "" the parent text: str - named: list[str] # the files its command prints, from the agent's start directory + named: list[str] # the files its command prints, from the start directory reads: list[dict[str, Any]] # their read events, in place before the command event moves: bool # the command may leave its shell in another directory result: str | None = None # "ok", "error" or "denied"; None while nothing came back @@ -86,7 +86,7 @@ class _Stream: """What a Claude stream-json trace carries from line to line.""" workdir: str | None - cwd: str # where each agent's shell starts, relative to workdir + cwd: str # where the Bash shell starts, relative to workdir by_id: dict[str, dict[str, Any]] = field(default_factory=dict) # tool_use id -> its event bash: dict[int, _BashCall] = field(default_factory=dict) # id() of a Bash call's command event -> the call @@ -160,26 +160,26 @@ def _deny(stream: _Stream, tool_id: Any) -> None: def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: - """Each Bash call's reads, once every result and denial is in. In each agent's order: + """Each Bash call's reads, once every result and denial is in. In the trace's order: - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``; - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file) drops its reads; - - after a call that was not denied and may have moved the agent's shell, only absolute reads are known; + - after a call that was not denied and may have moved its shell, only absolute reads are known, in + every agent: a subagent may share the parent's shell; - else the reads get ``rc`` 0, since a failure does not say which file failed (``cat a missing``). """ - moved: set[str] = set() # agents ("" the parent) whose shell may have left its start directory + moved = False # the shell may have left its start directory dropped: set[int] = set() for call in calls: failed = call.result == "error" and _failed_read(call.text, call.named, 1) for read, path in zip(call.reads, call.named, strict=True): - if failed or (call.agent in moved and not path.startswith("/")): + if failed or (moved and not path.startswith("/")): dropped.add(id(read)) elif call.result == "denied": read["denied"] = True elif call.result: read["rc"] = 0 - if call.moves and call.result != "denied": - moved.add(call.agent) + moved |= call.moves and call.result != "denied" if dropped: out.events[:] = [e for e in out.events if id(e) not in dropped] @@ -187,7 +187,7 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: """The events of one tool_use block: a Bash call's derived reads, then its own event.""" event = _tool_event(block, t, stream.workdir) - reads = _bash_call(event, agent, stream).reads if block.get("name") == _SHELL_TOOL else [] + reads = _bash_call(event, stream).reads if block.get("name") == _SHELL_TOOL else [] if agent: for e in (*reads, event): e["agent"] = agent @@ -198,11 +198,11 @@ def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: st stream.by_id[block["id"]] = event -def _bash_call(event: dict[str, Any], agent: str, stream: _Stream) -> _BashCall: +def _bash_call(event: dict[str, Any], stream: _Stream) -> _BashCall: """A Bash command's call, with a read event for each file it names; :func:`_settle_bash` keeps or drops them.""" named, moves = shell_reads_moved(str(event["text"]), stream.cwd) reads = [{"t": event["t"], "kind": "read", "path": relative_path(p, stream.workdir)} for p in named] - stream.bash[id(event)] = call = _BashCall(agent, str(event["text"]), named, reads, moves) + stream.bash[id(event)] = call = _BashCall(str(event["text"]), named, reads, moves) return call diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 4444701a1..4650b7d4e 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -145,16 +145,21 @@ def test_a_pushd_in_an_earlier_call_also_loses_the_directory(): assert _reads(events) == [] -def test_each_agent_keeps_its_own_shell_directory(): - events = _events( - _call("a", "cd sub"), _result("a"), - _call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), # a subagent's shell starts at the root - _call("p2", "cat AGENTS.md"), _result("p2"), # the parent's shell is in sub: not recorded - _call("s3", "cat AGENTS.md", agent="t1"), _result("s3"), # the parent's cd did not move the subagent - _call("s4", "cd x", agent="t1"), _result("s4"), - _call("s5", "cat AGENTS.md", agent="t1"), _result("s5")) +def test_a_parents_cd_drops_a_subagents_later_relative_read(): + # a subagent may share the parent's shell, so its relative read after the parent's cd is not known + events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), + _call("a", "cd sub"), _result("a"), + _call("s2", "cat AGENTS.md", agent="t1"), _result("s2"), + _call("s3", "cat /ws/sub/AGENTS.md", agent="t1"), _result("s3")) assert [(e["path"], e.get("agent")) for e in events if e["kind"] == "read"] == [ - ("AGENTS.md", "t1"), ("AGENTS.md", "t1")] + ("AGENTS.md", "t1"), ("sub/AGENTS.md", "t1")] + + +def test_a_subagents_cd_drops_the_parents_later_relative_read(): + events = _events(_call("p1", "cat AGENTS.md"), _result("p1"), + _call("s1", "cd sub", agent="t1"), _result("s1"), + _call("p2", "cat AGENTS.md"), _result("p2")) + assert [(e["path"], e.get("agent")) for e in events if e["kind"] == "read"] == [("AGENTS.md", None)] def test_a_subagent_bash_read_carries_the_agent(): From fe82755274828c74f36eaabd115fb588c51e0ca1 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:02:01 +0000 Subject: [PATCH 10/42] fix(repo): a refused Skill call is no use of the skill arms.skill_usage skipped a refused or failed SKILL.md read, but its `Skill` branch had no such check: a Skill tool call that a hook or the permissions refused (`denied`) still put the skill in `skills_used` and made `skill_invoked` true, though the skill never ran. The loop now skips a denied or failed event (`rc` not 0 or None) at the top, before either branch. The same class of bug was in uptake.arm_uptake: it skipped denied events, but a read of a file a `patch:` arm added that failed (`rc` 1, no such file) still counted in `uptake.reads`; it now takes only reads that showed the file (transcript.shown_read). Tests: a hook-denied and a permission-denied Skill call, and a failed read of an added file. The contract's skills_used and uptake text say so. Settles review comment 4445790411 on #369. Signed-off-by: Anton Volkov --- ...261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 ++-- dokimi/dokimi/repos/arms.py | 10 +++++----- dokimi/dokimi/repos/uptake.py | 6 ++++-- dokimi/tests/test_repo_skill_arms.py | 16 ++++++++++++++++ dokimi/tests/test_repo_uptake.py | 2 ++ 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index d1b83bc7b..70a22119a 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 1604e2d1c..8af7dcbb8 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -474,7 +474,7 @@ parent's and subagents' together, so a grader need not: A cell of an arm that installs skills also has a `skills` object in `result.json`: `measured`, `skills_installed` (the installed skill names), `skills_used` (the installed skills the agent called with the `Skill` tool or -whose `SKILL.md` it read, the read neither denied nor failed, in order of first use), `expected_skill` (the task's, +whose `SKILL.md` it read, the call or read neither denied nor failed, in order of first use), `expected_skill` (the task's, or `null`), `skill_invoked` (the expected skill was used; any installed skill when the task names none) and `skill_selected_correct` (the expected skill was the first one used; `null` when the task names none). When the trace could not @@ -488,7 +488,7 @@ For a `patch:` arm with `added` files or `mcp_servers`, `result.json` has an command events that name an added file as a workspace path (`tools/fmt`, `./tools/fmt`, `/eval/run/workspace/tools/fmt`, not `cd tools && ./fmt`; a file with no `/` only as `./name`, under the workspace path or after an interpreter (a bare `name` resolves through `PATH`), so -`npm run lint` does not name a `lint` file), `reads` the read events of one (a +`npm run lint` does not name a `lint` file), `reads` the read events of one that did not fail (a read of an added script counts as use), `mcp_calls` the calls of the servers' tools (`mcp____`), none of them a call the harness denied; `used` is whether any is non-zero, and `measured` is diff --git a/dokimi/dokimi/repos/arms.py b/dokimi/dokimi/repos/arms.py index 98f106b39..4e5bf8e5a 100644 --- a/dokimi/dokimi/repos/arms.py +++ b/dokimi/dokimi/repos/arms.py @@ -29,8 +29,6 @@ from dokimi import process -from .transcript import shown_read - SKILL_KINDS = ("skill", "skill-catalog") SKILLS_DIR = ".claude/skills" @@ -256,8 +254,8 @@ def skill_unmeasured(installed: list[str], expected: str | None) -> dict[str, An def skill_usage(events: list[dict[str, Any]], installed: list[str], expected: str | None) -> dict[str, Any]: - """Which installed skills the agent used: a ``Skill`` tool call naming it, or a read of its ``SKILL.md`` that was - neither refused nor failed. + """Which installed skills the agent used: a ``Skill`` tool call naming it, or a read of its ``SKILL.md``; neither + refused nor failed. ``skill_invoked``: the expected skill was used (any installed one when the task names none); ``skill_selected_correct``: the expected skill was the first one used, ``None`` without one. @@ -265,9 +263,11 @@ def skill_usage(events: list[dict[str, Any]], installed: list[str], expected: st used: list[str] = [] for e in events: name = None + if e.get("denied") or e.get("rc") not in (0, None): # a refused or failed call ran or showed no skill + continue if e.get("kind") == "tool" and e.get("name") == "Skill": name = str(e.get("skill") or "").split(":")[-1] # a plugin skill is plugin:name - elif shown_read(e): # a refused or failed read of a SKILL.md showed nothing of the skill + elif e.get("kind") == "read": parts = PurePosixPath(str(e.get("path") or "")).parts if len(parts) == 4 and "/".join(parts[:2]) == SKILLS_DIR and parts[3] == "SKILL.md": name = parts[2] diff --git a/dokimi/dokimi/repos/uptake.py b/dokimi/dokimi/repos/uptake.py index 9e00b20d9..a32cce968 100644 --- a/dokimi/dokimi/repos/uptake.py +++ b/dokimi/dokimi/repos/uptake.py @@ -23,7 +23,7 @@ Whether the harness starts them is the profile's business: a profile that starts none gives 0 calls, which is what the warning is for. -A call the harness denied (``denied``) is not counted. ``used`` is whether any +A call the harness denied (``denied``) is not counted, nor a read that failed. ``used`` is whether any count is non-zero; ``measured`` is false when the agent left no trace to count from (it never started, or its trace did not parse). """ @@ -35,6 +35,8 @@ from pathlib import Path from typing import Any +from .transcript import shown_read + MCP_CONFIG = ".mcp.json" @@ -79,7 +81,7 @@ def arm_uptake(events: list[dict[str, Any]] | None, files: list[str], servers: l kind = ev.get("kind") if kind == "command" and named is not None and named.search(str(ev.get("text", ""))): out["commands"] += 1 - elif kind == "read" and ev.get("path") in wanted: + elif shown_read(ev) and ev.get("path") in wanted: # a failed read (``rc`` 1) showed nothing out["reads"] += 1 elif kind == "tool" and prefixes and str(ev.get("name", "")).startswith(prefixes): out["mcp_calls"] += 1 diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py index 6a523b4ce..a378aa209 100644 --- a/dokimi/tests/test_repo_skill_arms.py +++ b/dokimi/tests/test_repo_skill_arms.py @@ -109,6 +109,22 @@ def test_a_refused_read_of_skill_md_is_no_use_of_the_skill(result, final): assert skill_usage(events, ["lint"], "lint")["skills_used"] == [] +@pytest.mark.parametrize("result, final", [ + ("PreToolUse:Skill hook error: no", {}), + ("Permission to use Skill has been denied", {"permission_denials": [{"tool_name": "Skill", "tool_use_id": "a"}]}), +]) +def test_a_refused_skill_call_is_no_use_of_the_skill(result, final): + lines = [{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "a", "name": "Skill", + "input": {"skill": "lint"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "a", "is_error": True, + "content": result}]}}, + {"type": "result", "result": "ok", **final}] + events = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws").events + assert [(e["name"], e.get("denied")) for e in events] == [("Skill", True)] + assert skill_usage(events, ["lint"], "lint")["skills_used"] == [] + assert skill_usage([{**events[0], "denied": False}], ["lint"], "lint")["skills_used"] == ["lint"] + + def test_a_failed_read_of_skill_md_is_no_use_of_the_skill(): failed = {"kind": "read", "path": ".claude/skills/lint/SKILL.md", "rc": 1} assert skill_usage([failed], ["lint"], "lint")["skills_used"] == [] diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index 73a439443..9d0c6f761 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -75,6 +75,8 @@ def test_reads_and_calls_of_the_arms_mcp_servers_count_others_do_not(): got = arm_uptake(events, ["tools/fmt"], ["code.graph"], WS) assert (got["reads"], got["mcp_calls"], got["commands"], got["used"], got["measured"]) == (1, 1, 0, True, True) assert arm_uptake([], ["tools/fmt"], [], WS)["used"] is False + failed = arm_uptake([{"kind": "read", "path": "tools/fmt", "rc": 1}], ["tools/fmt"], [], WS) # no such file + assert (failed["reads"], failed["used"]) == (0, False) unmeasured = arm_uptake(None, ["tools/fmt"], [], WS) assert unmeasured["measured"] is False and unmeasured["used"] is False From 0e16f14200c9710750a09b787f8774818e29f74a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:03:16 +0000 Subject: [PATCH 11/42] fix(repo): a failed cd-then-read command is no read `_failed_read` dropped the read of a failed command only when the command was one part. `cd sub && cat AGENTS.md` with no sub/AGENTS.md exits 1, yet its read of sub/AGENTS.md stayed with `rc` 0, so guidance_read, skills_used and reach's `fetched` counted a file the agent never saw. A one-file read after `cd` parts now counts as one read part: a non-zero exit means the read did not run (`cd sub && cat f`, the cd failed) or ran and failed (the exit after `;`, `&` or `||` is the read's), and either way the file was not shown. A list whose other part is not a `cd` (`make && cat f`, `cat f && cd sub`) keeps its read, as the exit does not say which part failed. The rule is shared, so a Codex or Copilot command gets the same fix. Tests: the reviewer's probe and five more cd forms, three lists that keep their read, and a Copilot row. The contract's guidance_read row and `fetched` bullet say so. Settles review comment 4446227568 on #369. Signed-off-by: Anton Volkov --- ...0261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 5 +++-- dokimi/dokimi/repos/cli_traces.py | 6 ++++-- dokimi/dokimi/repos/events.py | 3 ++- dokimi/tests/test_repo_cli_traces.py | 3 ++- dokimi/tests/test_repo_events_bash.py | 15 +++++++++++++++ 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 70a22119a..c6a475145 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none) gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 8af7dcbb8..361f0cf5d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -466,7 +466,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1074,7 +1074,8 @@ per scenario). read that was denied or failed is none). For the Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one - such read of one file and exits non-zero is not a read (the exit code of + such read of one file, alone or after `cd` commands (`cd sub && cat + AGENTS.md`), and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative read after an earlier Bash call of any agent (the parent's or a diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index faef86db2..c87860de0 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -667,12 +667,14 @@ def _codex_command_unstarted(out: Transcript, item: dict[str, Any], script: str, def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: - """A non-zero exit of a command that is one part naming one file: that file was not read. + """A non-zero exit of a command that is one part naming one file, alone or after ``cd`` parts + (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read. Otherwise the exit code does not say which read failed (``cat a missing`` prints ``a`` and exits 1), so the reads are kept. """ - return rc not in (None, 0) and len(reads) == 1 and len(_split_script(script)) == 1 + return rc not in (None, 0) and len(reads) == 1 and all( # each part before the last is a ``cd`` + part.split()[:1] == ["cd"] for part, _, _ in _split_script(script)[:-1]) class _CopilotRun: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 6f0f8c5bd..b40143665 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -163,7 +163,8 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: """Each Bash call's reads, once every result and denial is in. In the trace's order: - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``; - - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file) drops its reads; + - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file, after any + ``cd``: :func:`~dokimi.repos.cli_traces._failed_read`) drops its reads; - after a call that was not denied and may have moved its shell, only absolute reads are known, in every agent: a subagent may share the parent's shell; - else the reads get ``rc`` 0, since a failure does not say which file failed (``cat a missing``). diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index e0fe36c97..3ae9ef682 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -563,7 +563,8 @@ def test_copilot_command_exit_code_falls_back_to_success(done, rc): ("cat AGENTS.md", {"success": False}, set()), ("cat AGENTS.md", {"shellExecution": {"exitCode": 0}}, {"AGENTS.md"}), ("cat a; cat b", {"shellExecution": {"exitCode": 1}}, {"a", "b"}), # which part failed is not known -], ids=["exit-1", "success-false", "exit-0", "compound"]) + ("cd sub && cat AGENTS.md", {"shellExecution": {"exitCode": 1}}, set()), # the cd or the cat failed +], ids=["exit-1", "success-false", "exit-0", "compound", "cd-then-cat"]) def test_copilot_failed_bash_read_is_not_a_read(command, done, reads): raw = _jsonl( {"type": "tool.execution_start", diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 4650b7d4e..985dc1958 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -55,6 +55,21 @@ def test_a_failed_cat_of_one_file_is_no_read(): "output": "cat: AGENTS.md: No such file or directory", "output_truncated": False}] +@pytest.mark.parametrize("command", ["cd sub && cat AGENTS.md", "cd sub; cat AGENTS.md", "cd a && cd b && cat AGENTS.md", + "cd $(git rev-parse --show-toplevel) && cat /ws/AGENTS.md", + "cd /ws/sub || cat /ws/AGENTS.md", "cd sub & cat AGENTS.md"]) # the exit is the cat's +def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): + events = _events(_call("a", command), _result("a", "cat: AGENTS.md: No such file or directory", True)) + assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] + + +@pytest.mark.parametrize("command", ["cat AGENTS.md && cd sub", "make && cat AGENTS.md", "(cd sub) && cat AGENTS.md"]) +def test_a_failed_list_whose_other_part_is_not_a_cd_keeps_the_read(command): + # the failure may be the other part's: the file may have been shown + events = _events(_call("a", command), _result("a", "error", True)) + assert [(e["kind"], e["rc"]) for e in events] == [("read", 0), ("command", 1)] + + def test_a_failed_cat_of_two_files_keeps_both_reads(): events = _events(_call("a", "cat AGENTS.md missing.md"), _result("a", "# rules\ncat: missing.md: No such file", True)) assert [(e["path"], e["rc"]) for e in events if e["kind"] == "read"] == [("AGENTS.md", 0), ("missing.md", 0)] From 58920c4bcaf7a3b97f31019a1fecd1a46af2f528 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:04:06 +0000 Subject: [PATCH 12/42] fix(repo): a refused Bash read after a cd stays, denied After a `cd sub` that ran, a hook-refused `cat AGENTS.md` lost its read event: `_settle_bash` checked "the shell may have moved, so a relative read is not known" before it checked the denial, and dropped the read. Only the command event was left, so `tool_calls` was one short, and the module docstring's promise that a refused call's reads stay, `denied`, did not hold. The denial is now checked first. A refused call ran nothing, so its reads stay, `denied`, after a move too; a relative path is then taken from the start directory, and guidance_read, reach and skill use skip a denied read anyway. Test: a `cd sub`, then a `cat AGENTS.md` refused by a hook or by `permission_denials`, keeps the denied read, counts 3 tool calls and gives no guidance_read. Settles review comment 4446227577 on #369. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/events.py | 11 ++++++----- dokimi/tests/test_repo_events_bash.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index b40143665..e50ae60b0 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -24,7 +24,8 @@ next, so once any call ran a ``cd`` in that shell (not in a subshell, a pipeline or a background job), later relative reads are not recorded, the parent's and every subagent's alike: subagents may share the parent's shell, and dropping a read is the safe side either way. A refused call ran nothing: -it moves no shell, and its reads stay, ``denied``. The reads are settled when the trace ends, +it moves no shell, and its reads stay, ``denied``, after a move too (a relative path is then taken +from the start directory; ``guidance_read``, reach and skill use skip a denied read). The reads are settled when the trace ends, since a ``permission_denials`` entry comes only in the final result. Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` @@ -162,7 +163,7 @@ def _deny(stream: _Stream, tool_id: Any) -> None: def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: """Each Bash call's reads, once every result and denial is in. In the trace's order: - - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``; + - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``, after a move too; - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file, after any ``cd``: :func:`~dokimi.repos.cli_traces._failed_read`) drops its reads; - after a call that was not denied and may have moved its shell, only absolute reads are known, in @@ -174,10 +175,10 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: for call in calls: failed = call.result == "error" and _failed_read(call.text, call.named, 1) for read, path in zip(call.reads, call.named, strict=True): - if failed or (moved and not path.startswith("/")): - dropped.add(id(read)) - elif call.result == "denied": + if call.result == "denied": # kept after a move too, its path taken from the start directory read["denied"] = True + elif failed or (moved and not path.startswith("/")): + dropped.add(id(read)) elif call.result: read["rc"] = 0 moved |= call.moves and call.result != "denied" diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 985dc1958..f9aa51ed1 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -109,6 +109,17 @@ def test_a_refused_cd_moves_no_shell(refusal, reads): assert [e["kind"] for e in events] == ["command", *(["read"] if reads else []), "command"] +@pytest.mark.parametrize("refusal", [ + lambda: _result("b", "PreToolUse:Bash hook error: no", True), + lambda: _denials("b"), +]) +def test_a_refused_relative_read_after_a_cd_stays_denied(refusal): + events = _events(_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), refusal()) + assert [(e["kind"], e.get("denied")) for e in events] == [("command", None), ("read", True), ("command", True)] + got = process_metrics(events, lambda p: p.endswith("AGENTS.md")) + assert got["tool_calls"] == 3 and got["guidance_read"] == [] + + @pytest.mark.parametrize("command", ["cat AGENTS.md | head", "grep -n rule AGENTS.md", "cat AGENTS.md > copy.md"]) def test_a_pipe_a_search_or_a_redirect_is_no_read(command): assert _reads(_events(_call("a", command), _result("a"))) == [] From ff48bb15c32f79273c4064336e9f7d49b2f5d7e3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:05:59 +0000 Subject: [PATCH 13/42] fix(repo): eval, source and time cd move the Bash shell too `shell_reads_moved` did not set `moved` for `eval "cd sub"`, `source env.sh` or `time cd sub`, so after such a Claude Code Bash call a later relative `cat AGENTS.md` was still recorded as a read of the root AGENTS.md, though the shell may be elsewhere. `eval`, `source` and `.` now lose the directory as `pushd` / `popd` do: they may run a `cd` that is not seen, so the shell may have moved (and within one call, a relative read after them is not recorded). `time` before a directory builtin is skipped like `builtin` and `command`, so `time cd sub` is a `cd` (`command cd sub` already was). The parser is shared, so a Codex or Copilot command follows the same rule within the command. A `cd` in a shell function defined and then called is still not seen; the contract's "cannot show" list says so, and its guidance_read row and `fetched` bullet name the new forms. Tests: each form's `moved`, a later call's relative read, and reads within the call. Settles review comment 4446227585 on #369. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 11 +++--- dokimi/dokimi/repos/cli_traces.py | 36 ++++++++++--------- dokimi/tests/test_repo_events_bash.py | 16 +++++++-- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index c6a475145..2dcc0d1a9 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd` or `pushd` in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 361f0cf5d..23ee735f2 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -466,7 +466,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd`, `pushd` or `popd` in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1079,9 +1079,9 @@ per scenario). `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative read after an earlier Bash call of any agent (the parent's or a - subagent's, which may share its shell) ran a `cd` in its own shell (not in - a subshell, pipeline or background job, and not refused), so such a file - can be classed `loaded`; + subagent's, which may share its shell) ran a `cd`, `pushd`, `popd`, + `eval`, `source` or `.` in its own shell (not in a subshell, pipeline or + background job, and not refused), so such a file can be classed `loaded`; - `not_loaded`; - `error`: the scenario produced no usable answer: no answer, a non-zero exit, or an error result. @@ -1183,6 +1183,9 @@ observations of that run. It cannot show: - which directory a Claude Code subagent's Bash shell is in: the converter assumes it shares the parent's, so a `cd` in either drops the later relative reads of both (this only drops reads; no real trace was checked); +- a `cd` inside a shell function the agent defined and then called: the call + is not followed, so a later relative read is taken from the directory + before it (an `eval`, `source` or `.` is taken as moving the shell); - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: they appear with `observed: unknown`. A skill body is loaded on demand, so it diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index c87860de0..8f13f7dd4 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -49,8 +49,9 @@ _SYSTEM_BIN = {"/bin", "/usr/bin", "/usr/local/bin"} _ASSIGN = re.compile(r"[A-Za-z_]\w*=") # a NAME=value word: a variable set for the command that follows _KEYWORDS = {"if", "then", "else", "elif", "do", "while", "until", "!"} -_DIR_STACK = {"pushd", "popd"} -_DIRS = {"cd"} | _DIR_STACK # the builtins that change the directory +# after these builtins the directory is not known: a directory stack, or a script that may run a ``cd`` +_DIR_LOST = {"pushd", "popd", "eval", "source", "."} +_DIRS = {"cd"} | _DIR_LOST # the builtins that change, or may change, the directory _PIPE = re.compile(r"(? list[str]: script starts in, relative to the workspace root. ``cd`` moves it for the parts that follow: a relative path is joined to it and normalised (no ``.`` or ``..``). The directory is unknown (a relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a - ``pushd`` / ``popd``, ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` - after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain - holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the - files are not known). A subshell ``( ... )`` or `` `...` `` records no reads: its directory is not kept. A brace - group or an ``&&`` / ``||`` list that is backgrounded, or a piped group, runs in a subshell, so the directory is - restored after it; the reads of a + ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``), ``env -C`` / ``--chdir`` (the + command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a + ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been + skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records + no reads: its directory is not kept. A brace group or an ``&&`` / ``||`` list that is backgrounded, or a piped + group, runs in a subshell, so the directory is restored after it; the reads of a group whose output feeds a pipe after it is dropped, as a pipeline is not a read here; a group whose stdout is redirected to a file or /dev/null is dropped, as that output is not shown to the agent. ``--help`` and ``--version`` print and open no file. A function definition is skipped whole. Absolute paths are kept. @@ -101,8 +102,9 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: """:func:`shell_reads`, and whether the script may leave its shell in another directory: it ran a ``cd``, - ``pushd`` or ``popd`` in that shell, not in a subshell (``( ... )``, a pipeline or a backgrounded command, list - or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``, ``cd "$X" &``, ``cd sub | cat``).""" + ``pushd``, ``popd``, ``eval``, ``source`` or ``.`` in that shell, not in a subshell (``( ... )``, a pipeline + or a backgrounded command, list or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``, + ``cd "$X" &``, ``cd sub | cat``).""" reads, walk = _walk_script(script, cwd) return reads, walk.shell_moved @@ -228,7 +230,7 @@ def command(self, words: list[str], after: str, chained: bool = False) -> list[s name = _run_name(words[0], via_env) if name is None: return [] - if name in _DIR_STACK or (conditional and name == "cd"): + if name in _DIR_LOST or (conditional and name == "cd"): self.lose(after != "&") elif name == "cd": self.moved = True @@ -248,12 +250,12 @@ def resolve(self, f: str) -> str: def _run_name(name: str, via_env: bool) -> str | None: - """The name a command runs as, or None when it is a ``cd`` or directory-stack builtin that does not run.""" - if via_env and (name == "cd" or name in _DIR_STACK): # ``env cd x`` runs no builtin: nothing moves + """The name a command runs as, or None when it is a directory builtin (``cd``, ``pushd``, ``eval``, ...) that does not run.""" + if via_env and (name == "cd" or name in _DIR_LOST): # ``env cd x`` runs no builtin: nothing moves return None if "/" in name: # ``/bin/cat`` is cat; ``./cat`` or ``/bin/cd`` is some other program name = posixpath.basename(name) if posixpath.dirname(name) in _SYSTEM_BIN else "" - if name == "cd" or name in _DIR_STACK: + if name == "cd" or name in _DIR_LOST: return None return name @@ -436,13 +438,13 @@ def _program(word: str) -> str: def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes - directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin`` or ``command`` before a directory - builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped.""" + directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or ``time`` before a + directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped.""" via_env = chdir = False while words: if _ASSIGN.match(words[0]): words = words[1:] - elif words[0] in ("builtin", "command") and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program + elif words[0] in ("builtin", "command", "time") and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program words = words[1:] elif _program(words[0]) == "env": words, via_env = words[1:], True diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index f9aa51ed1..f4b995c40 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -153,6 +153,8 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("cd sub && { echo a; }", True), ('cd "$X" &', False), ('cd "$X" | cat', False), ("cd sub | cat", False), ("echo | cd sub", False), ('cd "$X" && make', True), ("cd $(pwd)", True), (r"cd a\|b", True), + ('eval "cd sub"', True), ('eval "$CMD"', True), ("source env.sh", True), (". env.sh", True), # may run a cd + ("time cd sub", True), ("command cd sub", True), ('eval "cd sub" &', False), ("env source env.sh", False), ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @@ -166,11 +168,21 @@ def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): assert _reads(events) == ["AGENTS.md"] -def test_a_pushd_in_an_earlier_call_also_loses_the_directory(): - events = _events(_call("a", "pushd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) +@pytest.mark.parametrize("first", ["pushd sub", 'eval "cd sub"', "source .venv/bin/activate", ". env.sh", + "time cd sub"]) +def test_a_pushd_eval_or_source_in_an_earlier_call_also_loses_the_directory(first): + events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == [] +@pytest.mark.parametrize("command, reads", [ + ("time cd sub && cat AGENTS.md", ["sub/AGENTS.md"]), ("command cd sub && cat AGENTS.md", ["sub/AGENTS.md"]), + ('eval "cd sub" && cat AGENTS.md', []), ("source env.sh && cat AGENTS.md", []), (". env.sh; cat AGENTS.md", []), +]) +def test_time_runs_a_cd_and_eval_or_source_loses_the_directory_within_a_call(command, reads): + assert _reads(_events(_call("a", command), _result("a"))) == reads + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From 61a34a3714b22fb474060e18b9adf9298e8eebae Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:46:32 +0000 Subject: [PATCH 14/42] docs(repo): tool_calls counts kept shell reads, denied ones too The contract's tool_calls row and the process_metrics docstring said it counts the read event of each file a shell command printed. It counts every event: a refused `cat AGENTS.md` keeps a denied read, and a failed `cat AGENTS.md missing.md` keeps a read for both paths, since the exit code does not say which file failed. Say so, so the count is not taken for evidence that a file was shown. Settles review comment 4446703136 on #369. Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/process_metrics.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 23ee735f2..63c319211 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -462,7 +462,7 @@ parent's and subagents' together, so a grader need not: | Key | Meaning | |---|---| -| `tool_calls` | every event: each tool call the agent made, and the `read` event of each file a shell command printed (see `guidance_read`) | +| `tool_calls` | every event: each tool call the agent made, and each shell-derived `read` event kept, a denied one or one of a failed multi-file read too (see `guidance_read` and §7) | | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index 331669a22..fd6336780 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -5,8 +5,8 @@ The framework counts what every repository asks of a trace: -- ``tool_calls``: every event, so every tool call of the parent and its subagents, and the ``read`` event of - each file a shell command printed; +- ``tool_calls``: every event, so every tool call of the parent and its subagents, and each ``read`` event kept + from a shell command, a denied one or one of a failed multi-file read (``cat a missing``) too; - ``commands``: the ``command`` events; - ``failed_commands``: commands not denied whose ``rc`` is neither ``0`` nor ``null`` (``null`` means no result came back, which is not a failure; a permission denial carries ``rc`` 1 but never ran); From 6d06087f7325fe15fad531150c26122679f3c87f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:47:59 +0000 Subject: [PATCH 15/42] fix(repo): a failed Skill call is no use of the skill skill_usage skips an event with a non-zero rc, but the Claude Code converter set rc on read and command events only. A Skill call whose tool_result is an error ("Skill lint could not be loaded") with no hook refusal or permission denial stayed a plain tool event, so the skill counted as used, invoked and correctly selected. _apply_result now sets rc (0, or 1 on an error) on a Skill tool event as it does on a read. No other consumer reads rc on a tool event (uptake's mcp_calls, tool_calls, reach and the consistency check do not), so only skill usage changes. A permission-denied Skill call now carries rc 1 next to denied, as a permission-denied command does. Settles review comment 4446703155 on #369. Signed-off-by: Anton Volkov --- ...261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/events.py | 4 ++-- dokimi/tests/test_repo_skill_arms.py | 16 +++++++++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 2dcc0d1a9..811b71a6b 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused `Skill` tool call, and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 63c319211..f6bf730bd 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -435,7 +435,8 @@ reads the upstream base (`base_sha`) and the workspace start commit 8 KB keeps its first and last 4 KB around a `[... N bytes omitted ...]` line, and `output_truncated` is `true`; build errors are usually at the end. - `rc` is what the harness reports. Where it reports only success or failure - (Claude Code), `rc` is `0` or `1`. It is `null` when no result came back, + (Claude Code), `rc` is `0` or `1`; a Claude Code `Skill` tool event carries it + too (`1`: the skill was not loaded). It is `null` when no result came back, for example when the run stopped at its turn or budget limit; a grader must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index e50ae60b0..42eb50682 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -135,8 +135,8 @@ def _apply_result(out: Transcript, block: dict[str, Any], stream: _Stream) -> No _deny(stream, tool_id) elif id(result_of) in stream.bash: stream.bash[id(result_of)].result = "error" if block.get("is_error") else "ok" - if result_of["kind"] == "read" and not denied: - # A read that failed (the file does not exist) shows nothing of the file. + if (result_of["kind"] == "read" or result_of.get("name") == "Skill") and not denied: + # A read that failed (the file does not exist) shows nothing of the file; a failed Skill call, no skill. result_of["rc"] = 1 if block.get("is_error") else 0 if result_of["kind"] == "command": # Claude Code reports success or failure only, with stdout and stderr merged into one text. diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py index a378aa209..89a104282 100644 --- a/dokimi/tests/test_repo_skill_arms.py +++ b/dokimi/tests/test_repo_skill_arms.py @@ -122,7 +122,21 @@ def test_a_refused_skill_call_is_no_use_of_the_skill(result, final): events = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws").events assert [(e["name"], e.get("denied")) for e in events] == [("Skill", True)] assert skill_usage(events, ["lint"], "lint")["skills_used"] == [] - assert skill_usage([{**events[0], "denied": False}], ["lint"], "lint")["skills_used"] == ["lint"] + # a permission denial also comes back as an error result: rc 1, as a denied command has + assert skill_usage([{**events[0], "denied": False, "rc": 0}], ["lint"], "lint")["skills_used"] == ["lint"] + + +@pytest.mark.parametrize("is_error, used", [(True, []), (False, ["lint"])]) +def test_a_failed_skill_call_is_no_use_of_the_skill(is_error, used): + lines = [{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "a", "name": "Skill", + "input": {"skill": "lint"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "a", "is_error": is_error, + "content": "Skill lint could not be loaded"}]}}, + {"type": "result", "result": "ok"}] + events = convert("claude-stream-json", "\n".join(json.dumps(x) for x in lines), "/ws").events + assert [(e["name"], e.get("denied"), e["rc"]) for e in events] == [("Skill", None, int(is_error))] + usage = skill_usage(events, ["lint"], "lint") + assert usage["skills_used"] == used and usage["skill_invoked"] is bool(used) def test_a_failed_read_of_skill_md_is_no_use_of_the_skill(): From 3a5eabfed8a5f7bf261a331f61454ccfd87356c6 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:48:48 +0000 Subject: [PATCH 16/42] fix(repo): a cd $(...) in a pipeline moves no Bash shell A part holding a command substitution goes down the walker's subshell branch, which marked the shell moved unless the part was backgrounded. So `cd $(pwd) | cat` set shell_moved although the cd ran in a pipeline stage, and the next Claude Code Bash call's `cat AGENTS.md` lost its relative read. Both branches that see a directory builtin with a computed operand now ask one helper, _in_own_shell, whether the part runs in the script's shell: not backgrounded, and holding no pipe outside quotes, escapes, $( ... ) and backticks (so `cd $(echo a | tr a b)` still moves). The computed-word branch of _Walk.words reuses _runs_dir_before_parens for its builtin test, which is the same test on a part without parentheses. Settles review comment 4446703160 on #369. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/cli_traces.py | 19 ++++++++++++------- dokimi/tests/test_repo_events_bash.py | 4 +++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 8f13f7dd4..6c2f2537b 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -129,7 +129,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: walk.here = None if in_subshell: # a subshell: its reads are not tracked if _runs_dir_before_parens(part): # ``cd $(dirname f)``: the cd is outside the parentheses - walk.lose(after != "&") + walk.lose(_in_own_shell(part, after)) continue walk.track_groups(_bare(part), after, out) words = walk.words(part, after) @@ -209,12 +209,8 @@ def words(self, part: str, after: str = "") -> list[str]: except ValueError: return [] if words is None: - rough = part.split() - while rough and (rough[0] in _KEYWORDS or rough[0] == "{"): - rough = rough[1:] - rough, via_env, _ = _command_words(rough) - if rough[:1] and rough[0] in _DIRS and not via_env: # ``builtin cd "$D"``: where it went is not known - self.lose(after != "&" and not _PIPE.search(re.sub(r"\\.", "", _bare(part)))) # ``\|`` is no pipe + if _runs_dir_before_parens(part): # ``builtin cd "$D"``: where it went is not known + self.lose(_in_own_shell(part, after)) return [] return words[1:] if words[:1] == ["{"] else words # a brace group runs in this shell: its cd and reads count @@ -285,6 +281,15 @@ def _runs_dir_before_parens(part: str) -> bool: return bool(words) and words[0] in _DIRS and not via_env +def _in_own_shell(part: str, after: str) -> bool: + """Whether ``part`` runs in the script's shell: not backgrounded by ``after``, nor a pipeline stage (a pipe + in quotes, escaped, ``\\|``, or in ``$( ... )`` or backticks is not one of ``part``'s).""" + text = re.sub(r"\\.", "", _bare(part)) + for _ in range(text.count("(") + 1): # the innermost ``( ... )`` first, then the ones around it + text = re.sub(r"\([^()]*\)|`[^`]*`", "", text) + return after != "&" and not _PIPE.search(text) + + def _computed(part: str) -> bool: """A shell expansion outside single quotes (``$x``, backticks), a pipe or redirect outside any quote, or a glob or brace expansion (``sub*``, ``AGENTS.*``, ``{a,b}``) outside quotes, whose files are not known here.""" diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index f4b995c40..fef8fb987 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -153,6 +153,7 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("cd sub && { echo a; }", True), ('cd "$X" &', False), ('cd "$X" | cat', False), ("cd sub | cat", False), ("echo | cd sub", False), ('cd "$X" && make', True), ("cd $(pwd)", True), (r"cd a\|b", True), + ("cd $(pwd) | cat", False), ("cd $(pwd) &", False), ("cd `pwd` | cat", False), ("cd $(echo a | tr a b)", True), ('eval "cd sub"', True), ('eval "$CMD"', True), ("source env.sh", True), (". env.sh", True), # may run a cd ("time cd sub", True), ("command cd sub", True), ('eval "cd sub" &', False), ("env source env.sh", False), ]) @@ -162,7 +163,8 @@ def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves @pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &", "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &", - 'cd "$X" &', 'cd "$X" | cat', "cd sub | cat", "echo | cd sub"]) + 'cd "$X" &', 'cd "$X" | cat', "cd sub | cat", "echo | cd sub", + "cd $(pwd) | cat", "cd $(pwd) &"]) def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == ["AGENTS.md"] From 13f858ba142f1de28dfc317a0b51d110ea980364 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 19:49:35 +0000 Subject: [PATCH 17/42] fix(repo): a failed time cd-then-read command is no read _failed_read drops the read of a failed command that is one read of one file after cd parts, but took a part for a cd only when its first word was `cd`. The shell walker also runs a cd after `time`, `command`, `builtin` or a NAME=value word, so a failing `time cd sub && cat AGENTS.md` kept its sub/AGENTS.md read at rc 0, and guidance, skill use, reach access and uptake counted a file that was never shown. The test now goes through _command_words, the walker's own helper, so the two agree: `time cd`, `command cd`, `builtin cd` and `X=1 cd` are a cd, and `env cd` (no builtin runs) is not, as before. The helper is shared by the Codex and Copilot readers too. Settles review comment 4446703173 on #369. Signed-off-by: Anton Volkov --- .../20261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 5 +++-- dokimi/dokimi/repos/cli_traces.py | 8 +++++++- dokimi/tests/test_repo_events_bash.py | 7 +++++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 811b71a6b..2f41b7c44 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f6bf730bd..d518dbe3a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -467,7 +467,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not `env cd`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1076,7 +1076,8 @@ per scenario). Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one such read of one file, alone or after `cd` commands (`cd sub && cat - AGENTS.md`), and exits non-zero is not a read (the exit code of + AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not + `env cd`), and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative read after an earlier Bash call of any agent (the parent's or a diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 6c2f2537b..10b7968f0 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -681,7 +681,13 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: exits 1), so the reads are kept. """ return rc not in (None, 0) and len(reads) == 1 and all( # each part before the last is a ``cd`` - part.split()[:1] == ["cd"] for part, _, _ in _split_script(script)[:-1]) + _is_cd(part) for part, _, _ in _split_script(script)[:-1]) + + +def _is_cd(part: str) -> bool: + """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``; not ``env cd``).""" + words, via_env, _ = _command_words(part.split()) + return words[:1] == ["cd"] and not via_env class _CopilotRun: diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index fef8fb987..e495e6469 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -57,13 +57,16 @@ def test_a_failed_cat_of_one_file_is_no_read(): @pytest.mark.parametrize("command", ["cd sub && cat AGENTS.md", "cd sub; cat AGENTS.md", "cd a && cd b && cat AGENTS.md", "cd $(git rev-parse --show-toplevel) && cat /ws/AGENTS.md", - "cd /ws/sub || cat /ws/AGENTS.md", "cd sub & cat AGENTS.md"]) # the exit is the cat's + "cd /ws/sub || cat /ws/AGENTS.md", "cd sub & cat AGENTS.md", # the exit is the cat's + "time cd sub && cat AGENTS.md", "command cd sub && cat AGENTS.md", + "builtin cd sub && cat AGENTS.md", "X=1 cd sub && cat AGENTS.md"]) def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): events = _events(_call("a", command), _result("a", "cat: AGENTS.md: No such file or directory", True)) assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] -@pytest.mark.parametrize("command", ["cat AGENTS.md && cd sub", "make && cat AGENTS.md", "(cd sub) && cat AGENTS.md"]) +@pytest.mark.parametrize("command", ["cat AGENTS.md && cd sub", "make && cat AGENTS.md", "(cd sub) && cat AGENTS.md", + "env cd sub && cat AGENTS.md"]) # ``env cd`` runs no builtin: some program failed def test_a_failed_list_whose_other_part_is_not_a_cd_keeps_the_read(command): # the failure may be the other part's: the file may have been shown events = _events(_call("a", command), _result("a", "error", True)) From 8ce0a93c267105d50c32e89bb0c15b2d0c0df18f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 20:11:43 +0000 Subject: [PATCH 18/42] fix(repo): follow a brace group past the $( ) and groups in it Settles the review point that `{ cd $(pwd); } | cat` and `{ cd $(pwd); } &` moved the Bash shell: the part holding `$( )` was skipped as a subshell before its `{` was seen, so the group's pipe or `&` had nothing to restore, and a later relative `cat AGENTS.md` was dropped. The splitter now gives each part its outer text, the text outside any `( ... )` or backticks. The walker tracks brace groups on that text before it skips a subshell part, and tells a `cd $(...)` by it. So: - a group whose first command holds `$( )` or backticks, or whose closing part does (`{ cd sub; } | grep $(x)`), opens and closes as any group; - a `cd` inside `$( )`, backticks or `( ... )` moves nothing here: `(true; cd sub)` and `echo $(true; cd sub; pwd)` took the inner `cd` as this shell's (the row for the backtick form is changed to the read it now keeps); - every brace of a part counts, in order: `{ { cd sub; }; }` moves the shell, `{ { cat a; }; }` reads `a`, and `{ cat a; } | { cat b; }` drops `a`, whose output feeds the pipe. The splitter moves to repos/shell_split.py, as cli_traces.py was at its line cap. The parser is shared, so Codex and Copilot commands change in the same way. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/architecture.md | 4 +- dokimi/dokimi/repos/cli_traces.py | 159 +++++------------- dokimi/dokimi/repos/shell_split.py | 104 ++++++++++++ dokimi/tests/test_repo_cli_traces.py | 13 +- dokimi/tests/test_repo_events_bash.py | 5 +- 6 files changed, 166 insertions(+), 121 deletions(-) create mode 100644 dokimi/dokimi/repos/shell_split.py diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 2f41b7c44..3179be098 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 5aafd5a88..52994397f 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -315,7 +315,9 @@ next release. `base` must resolve, and the guidance globs are evaluated at each base. - `dokimi/repos/events.py` — converts a harness trace to the `events.jsonl` run trace; `dokimi/repos/cli_traces.py` holds the Codex and - Copilot JSON converters, and `dokimi/repos/answers.py` the `answer_format` + Copilot JSON converters and the parser of the files a shell command prints + (`dokimi/repos/shell_split.py` splits the command into its parts), + and `dokimi/repos/answers.py` the `answer_format` conversions that turn a native review answer into a findings block. - `dokimi/repos/l0/` — `repo scan` (L0): checks the links, paths, identifiers, commands and build labels the guidance names against the same diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 10b7968f0..3649b0f40 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -26,6 +26,7 @@ from datetime import datetime from typing import Any +from .shell_split import split_script from .transcript import Transcript, _parse_time, clip_output, relative_path _BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) @@ -91,7 +92,8 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records - no reads: its directory is not kept. A brace group or an ``&&`` / ``||`` list that is backgrounded, or a piped + no reads, and a ``cd`` in it moves nothing here. A brace group (``{ cd $(pwd); }``, ``{ { cd x; }; }`` too) or an + ``&&`` / ``||`` list that is backgrounded, or a piped group, runs in a subshell, so the directory is restored after it; the reads of a group whose output feeds a pipe after it is dropped, as a pipeline is not a read here; a group whose stdout is redirected to a file or /dev/null is dropped, as that output is not shown to the agent. ``--help`` and @@ -104,7 +106,7 @@ def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: """:func:`shell_reads`, and whether the script may leave its shell in another directory: it ran a ``cd``, ``pushd``, ``popd``, ``eval``, ``source`` or ``.`` in that shell, not in a subshell (``( ... )``, a pipeline or a backgrounded command, list or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``, - ``cd "$X" &``, ``cd sub | cat``).""" + ``cd "$X" &``, ``cd sub | cat``, ``{ cd $(pwd); } &``).""" reads, walk = _walk_script(script, cwd) return reads, walk.shell_moved @@ -116,7 +118,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: # a line continuation is removed, not made a space (``cat AGENTS\.md`` reads AGENTS.md); a heredoc body # is input, not commands script = _HEREDOC.sub(lambda m: m.group(3), _CONTINUATION.sub("", script)) - for part, after, in_subshell in _split_script(script): + for part, after, in_subshell, outer in split_script(script): op, previous = previous, after chained = op in ("&&", "||") if not chained and walk.gated: # a cd after && or || may not have run: the next chain starts in no known directory @@ -127,11 +129,11 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: continue if op == "||" and walk.moved: walk.here = None + walk.track_groups(_bare(outer), after, out) # ``{ cd $(pwd); } | cat``: the group is in this shell if in_subshell: # a subshell: its reads are not tracked - if _runs_dir_before_parens(part): # ``cd $(dirname f)``: the cd is outside the parentheses - walk.lose(_in_own_shell(part, after)) + if _runs_dir(outer): # ``cd $(dirname f)``: the cd is outside the parentheses + walk.lose(_in_own_shell(outer, after)) continue - walk.track_groups(_bare(part), after, out) words = walk.words(part, after) if words: out.extend(walk.command(words, after, chained)) @@ -181,22 +183,27 @@ def lose(self, shell: bool = True) -> None: def track_groups(self, bare: str, after: str, out: list[str]) -> None: """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. A group is one command of the list around it: ``cd sub && { a; b; } &`` backgrounds the whole list. The reads of a group - whose output feeds a pipe after it, or a file, are dropped from ``out``: its output is not the agent's.""" - opened = _GROUP_OPEN.search(bare) - if opened: - piped = bool(_PIPE.search(bare[:opened.start()])) - self.groups.append(((self.here, self.shell_moved), piped, len(out), (self.list_start, self.listed))) - if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked - self.here = None - closed = _GROUP_CLOSE.search(bare) - if closed and self.groups: - before, piped, mark, (self.list_start, self.listed) = self.groups.pop() - after_close = bare[closed.start():] - redirects = bare[closed.end():] - if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): - del out[mark:] - if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell - self.here, self.shell_moved = before + whose output feeds a pipe after it, or a file, are dropped from ``out``: its output is not the agent's. Each + brace of ``bare`` counts, in order (``{ { cd x``, ``} | { cat a``).""" + braces = [(m.end() - 1, m) for m in _GROUP_OPEN.finditer(bare)] + [(m.end(), m) for m in _GROUP_CLOSE.finditer(bare)] + for _, m in sorted(braces, key=lambda b: b[0]): + if m.re is _GROUP_OPEN: + self.open_group(bare[:m.end() - 1], len(out)) + elif self.groups: + self.close_group(bare[m.start():], bare[m.end():], after, out) + + def open_group(self, before_brace: str, mark: int) -> None: + piped = bool(_PIPE.search(before_brace)) + self.groups.append(((self.here, self.shell_moved), piped, mark, (self.list_start, self.listed))) + if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked + self.here = None + + def close_group(self, after_close: str, redirects: str, after: str, out: list[str]) -> None: + before, piped, mark, (self.list_start, self.listed) = self.groups.pop() + if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): + del out[mark:] + if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell + self.here, self.shell_moved = before def words(self, part: str, after: str = "") -> list[str]: """The words of ``part``; none when it cannot be split or is computed (a computed ``cd`` loses the directory, @@ -209,10 +216,12 @@ def words(self, part: str, after: str = "") -> list[str]: except ValueError: return [] if words is None: - if _runs_dir_before_parens(part): # ``builtin cd "$D"``: where it went is not known + if _runs_dir(part): # ``builtin cd "$D"``: where it went is not known self.lose(_in_own_shell(part, after)) return [] - return words[1:] if words[:1] == ["{"] else words # a brace group runs in this shell: its cd and reads count + while words[:1] == ["{"]: # a brace group runs in this shell: its cd and reads count + words = words[1:] + return words def command(self, words: list[str], after: str, chained: bool = False) -> list[str]: """Move the directory for a ``cd``; the files a printing command reads. ``chained``: the command follows @@ -271,23 +280,20 @@ def _cd(here: str | None, args: list[str]) -> str | None: return None if here is None else posixpath.normpath(posixpath.join(here, target)) -def _runs_dir_before_parens(part: str) -> bool: - """Whether the command text before its first parenthesis runs a directory builtin, as ``cd $(...)`` does - (a subshell ``(cd x; ...)`` has no such text, so it changes nothing outside itself).""" - words = re.split(r"[()`]", part, maxsplit=1)[0].split() +def _runs_dir(outer: str) -> bool: + """Whether the outer text of a part (:class:`~dokimi.repos.shell_split.Part`) runs a directory builtin, as + ``cd $(...)`` does (a subshell ``(cd x; ...)`` has no such text, so it changes nothing outside itself).""" + words = outer.split() while words and (words[0] in _KEYWORDS or words[0] == "{"): words = words[1:] words, via_env, _ = _command_words(words) return bool(words) and words[0] in _DIRS and not via_env -def _in_own_shell(part: str, after: str) -> bool: - """Whether ``part`` runs in the script's shell: not backgrounded by ``after``, nor a pipeline stage (a pipe - in quotes, escaped, ``\\|``, or in ``$( ... )`` or backticks is not one of ``part``'s).""" - text = re.sub(r"\\.", "", _bare(part)) - for _ in range(text.count("(") + 1): # the innermost ``( ... )`` first, then the ones around it - text = re.sub(r"\([^()]*\)|`[^`]*`", "", text) - return after != "&" and not _PIPE.search(text) +def _in_own_shell(outer: str, after: str) -> bool: + """Whether the part with this outer text runs in the script's shell: not backgrounded by ``after``, nor a + pipeline stage (a pipe in quotes or escaped, ``\\|``, is not one).""" + return after != "&" and not _PIPE.search(re.sub(r"\\.", "", _bare(outer))) def _computed(part: str) -> bool: @@ -462,87 +468,6 @@ def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: return words, via_env, chdir -def _split_script(script: str) -> list[tuple[str, str, bool]]: - """``(part, operator after it, in a subshell)`` for each part of ``script`` split at unquoted ``&&``, - ``||``, ``;``, ``&`` and newlines (the operator is ``""`` after the last part). A ``#`` that starts a - word begins a comment, which runs to the end of the line and is dropped. A part inside ``( ... )``, - or holding a parenthesis, is in a subshell.""" - splitter = _Splitter(script) - i = 0 - while i < len(script): - i = splitter.step(i) - splitter.flush("") - return splitter.parts - - -class _Splitter: - """The state of ``_split_script``: the parts so far, the one being built, the open quote and parentheses.""" - - def __init__(self, script: str) -> None: - self.script = script - self.parts: list[tuple[str, str, bool]] = [] - self.cur: list[str] = [] - self.quote = "" - self.depth = 0 # open parentheses - self.paren = False # the part being built holds a parenthesis - self.tick = False # inside a backtick substitution, a subshell as ``$( )`` is - - def flush(self, op: str) -> None: - """End the part being built at ``op``.""" - text = "".join(self.cur).strip() - if text: - self.parts.append((text, op if op != "\n" else ";", self.paren or self.depth > 0 or self.tick)) - self.cur = [] - self.paren = self.depth > 0 or self.tick - - def step(self, i: int) -> int: - """Take the text at ``i``; the index to continue at.""" - script, c = self.script, self.script[i] - if self.quote: - return self._quoted(i) - if c in "'\"": - self.quote = c - self.cur.append(c) - elif c == "\\" and i + 1 < len(script): - self.cur.append(c + script[i + 1]) - return i + 2 - elif c == "#" and (i == 0 or script[i - 1] in " \t\n;&|()"): - end = script.find("\n", i) # the comment ends at the newline, which still splits - return len(script) if end < 0 else end - elif c == "`": - self.paren, self.tick = True, not self.tick - self.cur.append(c) - elif c in "()": - self.paren = True - self.depth = max(0, self.depth + (1 if c == "(" else -1)) - self.cur.append(c) - else: - return self._operator(i) - return i + 1 - - def _quoted(self, i: int) -> int: - script, c = self.script, self.script[i] - self.cur.append(c) - if c == "\\" and self.quote == '"' and i + 1 < len(script): - i += 1 - self.cur.append(script[i]) - elif c == self.quote: - self.quote = "" - return i + 1 - - def _operator(self, i: int) -> int: - script, c = self.script, self.script[i] - two = script[i:i + 2] - op = two if two in ("&&", "||") else c if c in ";\n" else "" - if c == "&" and not op and script[i - 1:i] not in (">", "<") and script[i + 1:i + 2] != ">": - op = "&" - if not op: - self.cur.append(c) - return i + 1 - self.flush(op) - return i + len(op) - - def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: out = Transcript() messages: list[str] = [] @@ -681,7 +606,7 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: exits 1), so the reads are kept. """ return rc not in (None, 0) and len(reads) == 1 and all( # each part before the last is a ``cd`` - _is_cd(part) for part, _, _ in _split_script(script)[:-1]) + _is_cd(part.text) for part in split_script(script)[:-1]) def _is_cd(part: str) -> bool: diff --git a/dokimi/dokimi/repos/shell_split.py b/dokimi/dokimi/repos/shell_split.py new file mode 100644 index 000000000..e634fbc12 --- /dev/null +++ b/dokimi/dokimi/repos/shell_split.py @@ -0,0 +1,104 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A shell script split into its parts, for :func:`~dokimi.repos.cli_traces.shell_reads`.""" + +from __future__ import annotations + +from typing import NamedTuple + + +class Part(NamedTuple): + """One part of a script: its text, the operator after it, whether it is in a subshell, and its outer text.""" + + text: str + after: str # ``&&``, ``||``, ``;``, ``&``, or ``""`` after the last part (a newline is ``;``) + in_subshell: bool # inside ``( ... )`` or backticks, or holding a parenthesis or a backtick + outer: str # the text in the script's own shell: what is inside ``( ... )`` or backticks is left out + + +def split_script(script: str) -> list[Part]: + """Each part of ``script``, split at unquoted ``&&``, ``||``, ``;``, ``&`` and newlines. A ``#`` that + starts a word begins a comment, which runs to the end of the line and is dropped. A part inside + ``( ... )``, or holding a parenthesis, is in a subshell.""" + splitter = _Splitter(script) + i = 0 + while i < len(script): + i = splitter.step(i) + splitter.flush("") + return splitter.parts + + +class _Splitter: + """The state of ``split_script``: the parts so far, the one being built, the open quote and parentheses.""" + + def __init__(self, script: str) -> None: + self.script = script + self.parts: list[Part] = [] + self.cur: list[str] = [] + self.outer: list[str] = [] # the text of the part being built that is outside any parenthesis or backtick + self.quote = "" + self.depth = 0 # open parentheses + self.paren = False # the part being built holds a parenthesis + self.tick = False # inside a backtick substitution, a subshell as ``$( )`` is + + def add(self, text: str) -> None: + self.cur.append(text) + if not self.depth and not self.tick: + self.outer.append(text) + + def flush(self, op: str) -> None: + """End the part being built at ``op``.""" + text = "".join(self.cur).strip() + if text: + in_subshell = self.paren or self.depth > 0 or self.tick + self.parts.append(Part(text, op if op != "\n" else ";", in_subshell, "".join(self.outer).strip())) + self.cur, self.outer = [], [] + self.paren = self.depth > 0 or self.tick + + def step(self, i: int) -> int: + """Take the text at ``i``; the index to continue at.""" + script, c = self.script, self.script[i] + if self.quote: + return self._quoted(i) + if c in "'\"": + self.quote = c + self.add(c) + elif c == "\\" and i + 1 < len(script): + self.add(c + script[i + 1]) + return i + 2 + elif c == "#" and (i == 0 or script[i - 1] in " \t\n;&|()"): + end = script.find("\n", i) # the comment ends at the newline, which still splits + return len(script) if end < 0 else end + elif c == "`": + self.paren, self.tick = True, not self.tick + self.cur.append(c) + elif c in "()": + self.paren = True + self.depth = max(0, self.depth + (1 if c == "(" else -1)) + self.cur.append(c) + else: + return self._operator(i) + return i + 1 + + def _quoted(self, i: int) -> int: + script, c = self.script, self.script[i] + self.add(c) + if c == "\\" and self.quote == '"' and i + 1 < len(script): + i += 1 + self.add(script[i]) + elif c == self.quote: + self.quote = "" + return i + 1 + + def _operator(self, i: int) -> int: + script, c = self.script, self.script[i] + two = script[i:i + 2] + op = two if two in ("&&", "||") else c if c in ";\n" else "" + if c == "&" and not op and script[i - 1:i] not in (">", "<") and script[i + 1:i + 2] != ">": + op = "&" + if not op: + self.add(c) + return i + 1 + self.flush(op) + return i + len(op) diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 3ae9ef682..e3dc68fa5 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -221,13 +221,24 @@ def test_shell_reads_joins_a_line_continuation_without_a_space(command, reads): @pytest.mark.parametrize(("command", "reads"), [ - ("echo `true; cd sub; pwd`; cat AGENTS.md", []), # as with $( ): a cd inside it is not this shell's + ("echo `true; cd sub; pwd`; cat AGENTS.md", ["AGENTS.md"]), # as with $( ): a cd inside it is not this shell's ("echo `pwd`; cat AGENTS.md", ["AGENTS.md"]), ("cd `dirname a/b`; cat AGENTS.md", []), ("echo \\`; cd sub; cat AGENTS.md", ["sub/AGENTS.md"])]) # an escaped backtick opens nothing def test_shell_reads_a_backtick_substitution_is_a_subshell(command, reads): assert shell_reads(command) == reads +@pytest.mark.parametrize(("command", "reads"), [ + ("echo $(true; cd sub; pwd); cat AGENTS.md", ["AGENTS.md"]), ("(true; cd sub); cat AGENTS.md", ["AGENTS.md"]), + ("{ cd $(pwd); } | cat; cat AGENTS.md", ["AGENTS.md"]), ("{ cd `pwd`; } & cat AGENTS.md", ["AGENTS.md"]), + ("{ (cd x); cd y; } | cat; cat AGENTS.md", ["AGENTS.md"]), ("{ cd $(pwd); }; cat AGENTS.md", []), + ("{ cd sub; } | grep $(echo x); cat AGENTS.md", ["AGENTS.md"]), + ("{ { cd sub; }; }; cat a", ["sub/a"]), ("{ { cat a; }; }", ["a"]), ("{ cat a; } | { cat b; }; cat c", ["c"])]) +def test_shell_reads_tracks_a_group_past_its_substitutions_and_the_groups_in_it(command, reads): + # a part holding $( ) or a nested { is still in this shell outside them: its braces open and close groups + assert shell_reads(command) == reads + + @pytest.mark.parametrize(("command", "reads"), [ ("cd sub && cat f & cat g", ["sub/f", "g"]), ("cd sub || cat f & cat g", ["g"]), ("true && cd sub & cat g", ["g"]), ("cd sub && cat f; cat g", ["sub/f", "sub/g"])]) diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index e495e6469..8461043ce 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -159,6 +159,9 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("cd $(pwd) | cat", False), ("cd $(pwd) &", False), ("cd `pwd` | cat", False), ("cd $(echo a | tr a b)", True), ('eval "cd sub"', True), ('eval "$CMD"', True), ("source env.sh", True), (". env.sh", True), # may run a cd ("time cd sub", True), ("command cd sub", True), ('eval "cd sub" &', False), ("env source env.sh", False), + ("{ cd $(pwd); } | cat", False), ("{ cd $(pwd); } &", False), ("{ cd `pwd`; } | cat", False), + ("{ source $(x); } &", False), ("{ { cd $(pwd); }; } | cat", False), ("(true; cd sub)", False), + ("{ cd $(pwd); }", True), ("{ { cd sub; }; }", True), ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @@ -167,7 +170,7 @@ def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves @pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &", "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &", 'cd "$X" &', 'cd "$X" | cat', "cd sub | cat", "echo | cd sub", - "cd $(pwd) | cat", "cd $(pwd) &"]) + "cd $(pwd) | cat", "cd $(pwd) &", "{ cd $(pwd); } | cat", "{ cd $(pwd); } &"]) def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == ["AGENTS.md"] From 3467d96444c01048756f41b06d91f554987f32fd Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 20:13:08 +0000 Subject: [PATCH 19/42] fix(repo): a directory builtin in a job or a pipe keeps the directory Settles the review point that `source env.sh & cat AGENTS.md` gave no read: a backgrounded `source` cannot change the foreground shell's directory, yet the walker forgot it, and the same held for a backgrounded or piped `eval`, `.`, `pushd`, `popd` and computed `cd` (`cd "$X" &`, `cd $(pwd) | cat`). `_Walk.lose()` had two meanings: "a directory builtin went somewhere unknown" and "the next chain starts where a gated cd may have left it". It now means the first. A builtin backgrounded by `&` or run as a pipeline stage ran in a subshell, so it leaves this shell's directory and its movement flag alone, and the reads after it in the same command are kept. The gated case sets the directory unknown itself, as before. A builtin in this shell (`source env.sh; cat AGENTS.md`, `pushd sub > /dev/null`) still loses the directory. The rule is the shared parser's, so it holds for Codex and Copilot commands too. Signed-off-by: Anton Volkov --- ...261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/dokimi/repos/cli_traces.py | 16 +++++++++------- dokimi/tests/test_repo_cli_traces.py | 10 ++++++++++ dokimi/tests/test_repo_events_bash.py | 10 +++++++++- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 3179be098..f24d0edf5 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command (`source env.sh & cat AGENTS.md` reads `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 3649b0f40..cebaf865e 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -88,7 +88,8 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: script starts in, relative to the workspace root. ``cd`` moves it for the parts that follow: a relative path is joined to it and normalised (no ``.`` or ``..``). The directory is unknown (a relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a - ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``), ``env -C`` / ``--chdir`` (the + ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``; none of these when backgrounded or + piped, as it then ran in a subshell: ``source env.sh & cat a`` reads a), ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records @@ -122,8 +123,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: op, previous = previous, after chained = op in ("&&", "||") if not chained and walk.gated: # a cd after && or || may not have run: the next chain starts in no known directory - walk.lose(False) - walk.gated = False + walk.here, walk.gated = None, False walk.chain(op, chained) if walk.in_function(_bare(part)): # a definition runs nothing continue @@ -175,10 +175,12 @@ def in_function(self, bare: str) -> bool: self.function = max(self.function + len(_GROUP_OPEN.findall(bare)) - len(_GROUP_CLOSE.findall(bare)), 0) return True - def lose(self, shell: bool = True) -> None: - """The directory is no longer known; ``shell``: because this shell may have moved (not a subshell's cd).""" - self.here, self.moved = None, True - self.shell_moved |= shell + def lose(self, own_shell: bool = True) -> None: + """A directory builtin went where it is not known. Not ``own_shell`` (backgrounded, or a pipeline stage): + it ran in a subshell, so this shell's directory stays known (``source env.sh & cat a`` reads a).""" + self.moved = True + if own_shell: + self.here, self.shell_moved = None, True def track_groups(self, bare: str, after: str, out: list[str]) -> None: """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. A group is diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index e3dc68fa5..f4b313651 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -207,6 +207,16 @@ def test_a_computed_directory_change_in_a_group_or_a_stack_loses_the_directory(c assert shell_reads(command, "sub") == [] +@pytest.mark.parametrize("command", [ + "source env.sh & cat AGENTS.md", 'eval "$CMD" & cat AGENTS.md', ". env.sh & cat AGENTS.md", + "source env.sh | cat; cat AGENTS.md", "pushd sub & cat AGENTS.md", 'cd "$X" & cat AGENTS.md', + "cd x | cat; cat AGENTS.md", "cd $(pwd) | cat; cat AGENTS.md", "cd `pwd` & cat AGENTS.md", +]) +def test_a_directory_builtin_in_a_job_or_a_pipeline_keeps_the_directory(command): + # it runs in a subshell: the cat after it is still in sub + assert shell_reads(command, "sub") == ["sub/AGENTS.md"] + + def test_shell_reads_start_in_the_given_directory(): assert shell_reads("cat AGENTS.md", "sub") == ["sub/AGENTS.md"] assert shell_reads("cd inner && cat ../x", "sub") == ["sub/x"] diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 8461043ce..7eb819316 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -170,7 +170,8 @@ def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves @pytest.mark.parametrize("first", ["cd sub &", "{ cd sub; } | cat", "{ cd sub; } &", "cd sub && make &", "cd sub && { echo a; echo b; } &", "cd sub && { cd x; cat a; } &", 'cd "$X" &', 'cd "$X" | cat', "cd sub | cat", "echo | cd sub", - "cd $(pwd) | cat", "cd $(pwd) &", "{ cd $(pwd); } | cat", "{ cd $(pwd); } &"]) + "cd $(pwd) | cat", "cd $(pwd) &", "{ cd $(pwd); } | cat", "{ cd $(pwd); } &", + "source env.sh &", 'eval "$CMD" | cat', ". env.sh &", "pushd sub &"]) def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == ["AGENTS.md"] @@ -191,6 +192,13 @@ def test_time_runs_a_cd_and_eval_or_source_loses_the_directory_within_a_call(com assert _reads(_events(_call("a", command), _result("a"))) == reads +@pytest.mark.parametrize("command", ["source env.sh & cat AGENTS.md", 'eval "$CMD" | cat; cat AGENTS.md', + 'cd "$X" & cat AGENTS.md']) +def test_a_source_eval_or_computed_cd_in_a_job_or_a_pipeline_keeps_the_calls_later_read(command): + events = _events(_call("a", command), _result("a"), _call("b", "cat README.md"), _result("b")) + assert _reads(events) == ["AGENTS.md", "README.md"] + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From e60774d690940ea00cece683310e5762ebed817f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 20:15:33 +0000 Subject: [PATCH 20/42] fix(repo): a failed read naming two files keeps the one it knows Settles the review point that `_failed_read` decided "one file" from the number of derived reads. The walker leaves out a relative operand when its directory is not known, so in `/ws` a failed `cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md` gave one read, `/ws/AGENTS.md`, and dropped it, though cat printed the guidance and exited 1 for the missing file. The walker now records how many files the last printing command named (its operands, past the options and the prefix words the walker skips), and `_failed_read` drops the reads only when that count is one. A failed `cd -; cat /ws/AGENTS.md` still gives no read. The rule is shared: the new rows cover a Claude Code Bash call, a Codex command and a Copilot bash command. Signed-off-by: Anton Volkov --- .../20261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 7 ++++--- dokimi/dokimi/repos/cli_traces.py | 9 ++++++--- dokimi/dokimi/repos/events.py | 2 +- dokimi/tests/test_repo_cli_traces.py | 9 +++++++-- dokimi/tests/test_repo_events_bash.py | 8 ++++++++ 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index f24d0edf5..c4f9f49d5 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read; a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command (`source env.sh & cat AGENTS.md` reads `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command (`source env.sh & cat AGENTS.md` reads `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d518dbe3a..9d6209c87 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -467,7 +467,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read of one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not `env cd`), and fails is no read. Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1075,10 +1075,11 @@ per scenario). read that was denied or failed is none). For the Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one - such read of one file, alone or after `cd` commands (`cd sub && cat + such read naming one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not `env cd`), and exits non-zero is not a read (the exit code of - `cat a missing` does not say which file failed, so both stay reads). A read + `cat a missing` does not say which file failed, so both stay reads, as + the known one does when the other is at a directory not known). A read through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative read after an earlier Bash call of any agent (the parent's or a subagent's, which may share its shell) ran a `cd`, `pushd`, `popd`, diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index cebaf865e..35bf914f5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -156,6 +156,7 @@ def __init__(self, here: str | None) -> None: self.list_start: tuple[str | None, bool] = (here, False) # (here, shell_moved) where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition + self.operands: int | None = None # how many files the last printing command named def chain(self, op: str, chained: bool) -> None: """A list of more than one command ended by ``&`` ran in a subshell, so this shell did not move: ``cd sub && @@ -248,6 +249,7 @@ def command(self, words: list[str], after: str, chained: bool = False) -> list[s if _asks_for_help(words[1:]): # ``cat --help AGENTS.md`` prints the help and opens no file return [] files = _sed_files(words[1:]) if name == "sed" else _print_files(name, words[1:]) + self.operands = len(files) # all it names, those at an unknown directory too # ``env -C dir cat f`` reads dir/f, which is not known: only an absolute path is the same return [self.resolve(f) for f in files if f.startswith("/") or (self.here is not None and not chdir)] return [] @@ -605,10 +607,11 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read. Otherwise the exit code does not say which read failed (``cat a missing`` prints ``a`` and - exits 1), so the reads are kept. + exits 1), so the reads are kept: also when only one of the files is known (``cd -; cat missing /ws/a``). """ - return rc not in (None, 0) and len(reads) == 1 and all( # each part before the last is a ``cd`` - _is_cd(part.text) for part in split_script(script)[:-1]) + if rc in (None, 0) or not reads or not all(_is_cd(part.text) for part in split_script(script)[:-1]): + return False + return _walk_script(script, "")[1].operands == 1 # the printer's own operands, not the reads kept of them def _is_cd(part: str) -> bool: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 42eb50682..b62ab0000 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -164,7 +164,7 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: """Each Bash call's reads, once every result and denial is in. In the trace's order: - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``, after a move too; - - a call that failed and was one read of one file (``cat AGENTS.md`` of a missing file, after any + - a call that failed and was one read naming one file (``cat AGENTS.md`` of a missing file, after any ``cd``: :func:`~dokimi.repos.cli_traces._failed_read`) drops its reads; - after a call that was not denied and may have moved its shell, only absolute reads are known, in every agent: a subagent may share the parent's shell; diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index f4b313651..ea678838b 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -520,9 +520,12 @@ def cmd(i, script): {"type": "item.started", "item": cmd("b", "cat b.md; false")}, {"type": "item.completed", "item": {**cmd("b", "cat b.md; false"), "exit_code": 1}}, {"type": "item.completed", "item": {**cmd("c", "head c.md d.md"), "exit_code": 1}}, + # two operands, of which only the absolute one is known after ``cd -``: it may be the one printed + {"type": "item.started", "item": cmd("d", "cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md")}, + {"type": "item.completed", "item": {**cmd("d", "cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md"), "exit_code": 1}}, ) reads = [e["path"] for e in convert("codex-json", raw, None).events if e["kind"] == "read"] - assert reads == ["sub/AGENTS.md", "sub/CLAUDE.md", "b.md", "c.md", "d.md"] + assert reads == ["sub/AGENTS.md", "sub/CLAUDE.md", "b.md", "c.md", "d.md", "/ws/AGENTS.md"] def test_answer_format_lists_agree(): @@ -585,7 +588,9 @@ def test_copilot_command_exit_code_falls_back_to_success(done, rc): ("cat AGENTS.md", {"shellExecution": {"exitCode": 0}}, {"AGENTS.md"}), ("cat a; cat b", {"shellExecution": {"exitCode": 1}}, {"a", "b"}), # which part failed is not known ("cd sub && cat AGENTS.md", {"shellExecution": {"exitCode": 1}}, set()), # the cd or the cat failed -], ids=["exit-1", "success-false", "exit-0", "compound", "cd-then-cat"]) + # cat names two files, one at a directory not known: the known one may be what it printed + ("cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md", {"shellExecution": {"exitCode": 1}}, {"/ws/AGENTS.md"}), +], ids=["exit-1", "success-false", "exit-0", "compound", "cd-then-cat", "two-operands-one-known"]) def test_copilot_failed_bash_read_is_not_a_read(command, done, reads): raw = _jsonl( {"type": "tool.execution_start", diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 7eb819316..79f5af6df 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -78,6 +78,14 @@ def test_a_failed_cat_of_two_files_keeps_both_reads(): assert [(e["path"], e["rc"]) for e in events if e["kind"] == "read"] == [("AGENTS.md", 0), ("missing.md", 0)] +@pytest.mark.parametrize("command", ["cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md", + "cd -; head -n 5 missing.md /ws/AGENTS.md"]) +def test_a_failed_read_of_two_files_keeps_the_one_known_after_a_lost_directory(command): + # the relative operand is at a directory not known, so no read of it: cat still named two files + events = _events(_call("a", command), _result("a", "# rules\ncat: missing.md: No such file", True)) + assert [(e["path"], e["rc"]) for e in events if e["kind"] == "read"] == [("AGENTS.md", 0)] + + @pytest.mark.parametrize("refusal", [ lambda: _result("a", "PreToolUse:Bash hook error: shell background", True), lambda: {"type": "result", "result": "ok", "permission_denials": [{"tool_name": "Bash", "tool_use_id": "a"}]}, From 9515e0b97ca92c9f9b30d2c84daaddc078635f76 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 20:41:55 +0000 Subject: [PATCH 21/42] fix(repo): a builtin in a job or a pipe leaves a later || its directory Settles the review point that `source env.sh & false || cat AGENTS.md` gave no read: the backgrounded `source` kept the foreground directory but still set the walker's chain flag `moved`, so the `||` branch after it dropped the directory. A piped `source env.sh | false || ...`, a backgrounded `cd`, `pushd`, `eval` or `.`, a computed `cd` in a job or a pipe, and a backgrounded or piped brace group or list did the same. A directory builtin now sets `moved` only when it runs in this shell, as `shell_moved` was already set. The flag is saved and restored with the directory and `shell_moved` (one `_Spot`) at a backgrounded list and at a piped or backgrounded brace group. A builtin in this shell (`source env.sh; false || cat AGENTS.md`) still loses the directory for the `||` branch. The parser is shared, so this holds for Codex and Copilot commands too. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/dokimi/repos/cli_traces.py | 38 +++++++++++-------- dokimi/tests/test_repo_cli_traces.py | 17 +++++++++ dokimi/tests/test_repo_events_bash.py | 3 +- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index c4f9f49d5..b49c2575b 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command (`source env.sh & cat AGENTS.md` reads `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 35bf914f5..6ceaaab8a 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -89,7 +89,7 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: relative path is joined to it and normalised (no ``.`` or ``..``). The directory is unknown (a relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``; none of these when backgrounded or - piped, as it then ran in a subshell: ``source env.sh & cat a`` reads a), ``env -C`` / ``--chdir`` (the + piped, as it then ran in a subshell: ``source env.sh & false || cat a`` reads a), ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records @@ -141,32 +141,42 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: return out, walk +_Spot = tuple[str | None, bool, bool] # (here, shell_moved, moved): what the end of a subshell restores + + class _Walk: """Where ``shell_reads`` is in the script: the directory (``None``: unknown), whether a ``cd`` was seen, whether the script's own shell may have moved, and the open brace groups.""" def __init__(self, here: str | None) -> None: self.here = here - self.moved = False # a cd was seen: after ||, the directory is not known + self.moved = False # a cd was seen in this shell: after ||, the directory is not known self.shell_moved = False # a directory builtin ran in the script's shell, not in a subshell self.gated = False # a cd after && or || was seen in this chain: it may not have run for the next chain - # the open brace groups: ((here, shell_moved) before it, in a pipeline, index in the reads where it began, - # the enclosing && / || list's (list_start, listed), which the group's own separators do not end) - self.groups: list[tuple[tuple[str | None, bool], bool, int, tuple[tuple[str | None, bool], bool]]] = [] - self.list_start: tuple[str | None, bool] = (here, False) # (here, shell_moved) where the && / || list began + # the open brace groups: the spot before it, in a pipeline, index in the reads where it began, the + # enclosing && / || list's (list_start, listed), which the group's own separators do not end + self.groups: list[tuple[_Spot, bool, int, tuple[_Spot, bool]]] = [] + self.list_start: _Spot = (here, False, False) # the spot where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition self.operands: int | None = None # how many files the last printing command named + def spot(self) -> _Spot: + return self.here, self.shell_moved, self.moved + + def back(self, spot: _Spot) -> None: + """Back to ``spot`` after a subshell: what ran in it moved neither this shell nor its ``||`` fallbacks.""" + self.here, self.shell_moved, self.moved = spot + def chain(self, op: str, chained: bool) -> None: """A list of more than one command ended by ``&`` ran in a subshell, so this shell did not move: ``cd sub && cat f & cat g`` reads sub/f and g (a lone ``cd x &`` and a ``{ ...; } &`` group are handled where they end).""" if op == "&" and self.listed: - self.here, self.shell_moved = self.list_start + self.back(self.list_start) if chained: self.listed = True else: - self.list_start, self.listed = (self.here, self.shell_moved), False + self.list_start, self.listed = self.spot(), False def in_function(self, bare: str) -> bool: """Whether ``bare`` is part of a function definition (``f() { cd x; cat a; }``): its body is skipped, as @@ -178,10 +188,9 @@ def in_function(self, bare: str) -> bool: def lose(self, own_shell: bool = True) -> None: """A directory builtin went where it is not known. Not ``own_shell`` (backgrounded, or a pipeline stage): - it ran in a subshell, so this shell's directory stays known (``source env.sh & cat a`` reads a).""" - self.moved = True + it ran in a subshell, so this shell's directory stays known (``source env.sh & false || cat a`` reads a).""" if own_shell: - self.here, self.shell_moved = None, True + self.here, self.shell_moved, self.moved = None, True, True def track_groups(self, bare: str, after: str, out: list[str]) -> None: """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. A group is @@ -197,7 +206,7 @@ def track_groups(self, bare: str, after: str, out: list[str]) -> None: def open_group(self, before_brace: str, mark: int) -> None: piped = bool(_PIPE.search(before_brace)) - self.groups.append(((self.here, self.shell_moved), piped, mark, (self.list_start, self.listed))) + self.groups.append((self.spot(), piped, mark, (self.list_start, self.listed))) if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked self.here = None @@ -206,7 +215,7 @@ def close_group(self, after_close: str, redirects: str, after: str, out: list[st if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): del out[mark:] if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell - self.here, self.shell_moved = before + self.back(before) def words(self, part: str, after: str = "") -> list[str]: """The words of ``part``; none when it cannot be split or is computed (a computed ``cd`` loses the directory, @@ -241,10 +250,9 @@ def command(self, words: list[str], after: str, chained: bool = False) -> list[s if name in _DIR_LOST or (conditional and name == "cd"): self.lose(after != "&") elif name == "cd": - self.moved = True self.gated |= chained if after != "&": # a backgrounded cd runs in a subshell: this shell stays where it was - self.here, self.shell_moved = _cd(self.here, words[1:]), True + self.here, self.shell_moved, self.moved = _cd(self.here, words[1:]), True, True elif name in _PRINTERS: if _asks_for_help(words[1:]): # ``cat --help AGENTS.md`` prints the help and opens no file return [] diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index ea678838b..783fd05d8 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -217,6 +217,23 @@ def test_a_directory_builtin_in_a_job_or_a_pipeline_keeps_the_directory(command) assert shell_reads(command, "sub") == ["sub/AGENTS.md"] +@pytest.mark.parametrize("command, reads", [ + ("source env.sh & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("source env.sh | false || cat AGENTS.md", ["sub/AGENTS.md"]), + ('eval "$X" & false || cat AGENTS.md', ["sub/AGENTS.md"]), (". env.sh & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("pushd x & false || cat AGENTS.md", ["sub/AGENTS.md"]), ("cd x & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ('cd "$X" | false || cat AGENTS.md', ["sub/AGENTS.md"]), ("cd $(pwd) & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("{ cd x; } & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("{ cd x; } | cat; false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("cd x && make & false || cat AGENTS.md", ["sub/AGENTS.md"]), + ("source env.sh; false || cat AGENTS.md", []), ("cd x; false || cat AGENTS.md", []), + ("{ cd x; }; false || cat AGENTS.md", []), ("cd x; source env.sh & false || cat AGENTS.md", []), +]) +def test_only_a_directory_builtin_in_this_shell_loses_the_directory_after_a_later_or(command, reads): + # the || branch runs where a failed cd left this shell; one in a job or a pipe left it alone + assert shell_reads(command, "sub") == reads + + def test_shell_reads_start_in_the_given_directory(): assert shell_reads("cat AGENTS.md", "sub") == ["sub/AGENTS.md"] assert shell_reads("cd inner && cat ../x", "sub") == ["sub/x"] diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 79f5af6df..7bfa63a30 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -201,7 +201,8 @@ def test_time_runs_a_cd_and_eval_or_source_loses_the_directory_within_a_call(com @pytest.mark.parametrize("command", ["source env.sh & cat AGENTS.md", 'eval "$CMD" | cat; cat AGENTS.md', - 'cd "$X" & cat AGENTS.md']) + 'cd "$X" & cat AGENTS.md', "source env.sh & false || cat AGENTS.md", + "source env.sh | false || cat AGENTS.md"]) def test_a_source_eval_or_computed_cd_in_a_job_or_a_pipeline_keeps_the_calls_later_read(command): events = _events(_call("a", command), _result("a"), _call("b", "cat README.md"), _result("b")) assert _reads(events) == ["AGENTS.md", "README.md"] From 7ba6c6a512f572a0ce9182d2517f77f58fddfec7 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 20:44:41 +0000 Subject: [PATCH 22/42] fix(repo): a quoted or escaped directory builtin name is the builtin Settles the review point that `"source" "$ENV"` moved nothing: a part with an expansion is not split by shlex, and the walker matched its blank-split first word, `"source"` with its quotes, against the builtins. A quoted or escaped `cd`, `pushd`, `popd`, `eval` or `.` with a computed operand (`'cd' "$D"`, `\eval "$X"`, `c"d" "$D"`), and a quoted `builtin`, `command`, `time` or `env` before one, were missed the same way, so a later relative read in the same command or the next Claude Code Bash call was taken at the old directory. The failed-read rule's `cd` test had the same gap: a failed `"cd" sub && cat AGENTS.md` kept its read. A new `_unquoted` helper removes each blank-split word's quotes and backslash escapes, as shlex does, without expanding anything (`"$D"` stays `$D`). `_runs_dir` and `_is_cd` pass their words through it before `_command_words` skips the prefixes; the words of a part shlex does split are already unquoted, so they are not unquoted twice (`'\cd'` names a program `\cd`, not the builtin). Keywords are still matched before unquoting: a quoted `"if"` is no keyword. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 +-- dokimi/dokimi/repos/cli_traces.py | 25 +++++++++++++++---- dokimi/tests/test_repo_cli_traces.py | 19 ++++++++++++++ dokimi/tests/test_repo_events_bash.py | 9 +++++-- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index b49c2575b..d4679c217 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), and a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 9d6209c87..b986ea461 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -467,7 +467,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one) in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, a quoted or escaped `"cd"` or `\cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one), its name quoted or escaped too (`"source" "$ENV"`, `\cd "$D"`), in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1076,7 +1076,7 @@ per scenario). Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one such read naming one file, alone or after `cd` commands (`cd sub && cat - AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd` too, not + AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, `"cd"` too, not `env cd`), and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads, as the known one does when the other is at a directory not known). A read diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 6ceaaab8a..5ae8e5106 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -296,9 +296,9 @@ def _runs_dir(outer: str) -> bool: """Whether the outer text of a part (:class:`~dokimi.repos.shell_split.Part`) runs a directory builtin, as ``cd $(...)`` does (a subshell ``(cd x; ...)`` has no such text, so it changes nothing outside itself).""" words = outer.split() - while words and (words[0] in _KEYWORDS or words[0] == "{"): + while words and (words[0] in _KEYWORDS or words[0] == "{"): # a quoted ``"if"`` is no keyword: skipped unquoted words = words[1:] - words, via_env, _ = _command_words(words) + words, via_env, _ = _command_words(_unquoted(words)) return bool(words) and words[0] in _DIRS and not via_env @@ -459,10 +459,25 @@ def _program(word: str) -> str: return posixpath.basename(word) if posixpath.dirname(word) in ("", *_SYSTEM_BIN) else "" +def _unquoted(words: list[str]) -> list[str]: + """Words split at blanks, each with its quotes and backslash escapes removed and nothing expanded (``"cd"``, + ``'cd'``, ``\\cd`` and ``c"d"`` are ``cd``; ``"$D"`` is ``$D``), as ``shlex.split`` gives them; a word + whose quotes do not close here (``"a b"`` split at its blank) stays as it is.""" + out = [] + for word in words: + try: + split = shlex.split(word) + except ValueError: + split = [] + out.append(split[0] if len(split) == 1 else word) + return out + + def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or ``time`` before a - directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped.""" + directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped. + ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" via_env = chdir = False while words: if _ASSIGN.match(words[0]): @@ -623,8 +638,8 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: def _is_cd(part: str) -> bool: - """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``; not ``env cd``).""" - words, via_env, _ = _command_words(part.split()) + """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``, ``"cd"``; not ``env cd``).""" + words, via_env, _ = _command_words(_unquoted(part.split())) return words[:1] == ["cd"] and not via_env diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 783fd05d8..a099bec59 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -207,6 +207,25 @@ def test_a_computed_directory_change_in_a_group_or_a_stack_loses_the_directory(c assert shell_reads(command, "sub") == [] +@pytest.mark.parametrize("command", [ + '"source" "$ENV"; cat AGENTS.md', "'cd' \"$D\" && cat AGENTS.md", r'\eval "$X"; cat AGENTS.md', + '"." env.sh; cat AGENTS.md', 'c"d" "$D"; cat AGENTS.md', '"pushd" "$D"; cat AGENTS.md', + '"builtin" cd "$D"; cat AGENTS.md', 'builtin "cd" "$D"; cat AGENTS.md', '"cd" $(pwd); cat AGENTS.md', + "'cd' `pwd`; cat AGENTS.md", r'\pushd sub; cat AGENTS.md', +]) +def test_a_quoted_or_escaped_directory_builtin_loses_the_directory(command): + # quoting or escaping a builtin's name still runs the builtin; nothing in it is expanded to see that + assert shell_reads(command, "sub") == [] + + +@pytest.mark.parametrize("command", [ + '"env" cd "$D"; cat AGENTS.md', "'\\cd' \"$D\"; cat AGENTS.md", '"./cd" "$D"; cat AGENTS.md', + '"source" "$E" & cat AGENTS.md', '"cd" "$D" | cat; cat AGENTS.md', +]) +def test_a_quoted_name_that_runs_no_builtin_in_this_shell_keeps_the_directory(command): + assert shell_reads(command, "sub") == ["sub/AGENTS.md"] + + @pytest.mark.parametrize("command", [ "source env.sh & cat AGENTS.md", 'eval "$CMD" & cat AGENTS.md', ". env.sh & cat AGENTS.md", "source env.sh | cat; cat AGENTS.md", "pushd sub & cat AGENTS.md", 'cd "$X" & cat AGENTS.md', diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 7bfa63a30..1b31ae6b8 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -59,7 +59,9 @@ def test_a_failed_cat_of_one_file_is_no_read(): "cd $(git rev-parse --show-toplevel) && cat /ws/AGENTS.md", "cd /ws/sub || cat /ws/AGENTS.md", "cd sub & cat AGENTS.md", # the exit is the cat's "time cd sub && cat AGENTS.md", "command cd sub && cat AGENTS.md", - "builtin cd sub && cat AGENTS.md", "X=1 cd sub && cat AGENTS.md"]) + "builtin cd sub && cat AGENTS.md", "X=1 cd sub && cat AGENTS.md", + '"cd" sub && cat AGENTS.md', r"\cd sub && cat AGENTS.md", + "builtin 'cd' sub && cat AGENTS.md"]) # a quoted or escaped name is the builtin def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): events = _events(_call("a", command), _result("a", "cat: AGENTS.md: No such file or directory", True)) assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] @@ -170,6 +172,8 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("{ cd $(pwd); } | cat", False), ("{ cd $(pwd); } &", False), ("{ cd `pwd`; } | cat", False), ("{ source $(x); } &", False), ("{ { cd $(pwd); }; } | cat", False), ("(true; cd sub)", False), ("{ cd $(pwd); }", True), ("{ { cd sub; }; }", True), + ('"source" "$ENV"', True), ("'cd' \"$D\"", True), (r'\eval "$X"', True), ('"." env.sh', True), # quoted names + ('"cd" "$D" &', False), ('"source" "$E" | cat', False), ('"env" cd "$D"', False), ("'\\cd' \"$D\"", False), ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @@ -186,7 +190,8 @@ def test_a_cd_in_a_job_or_a_pipeline_keeps_the_next_calls_relative_read(first): @pytest.mark.parametrize("first", ["pushd sub", 'eval "cd sub"', "source .venv/bin/activate", ". env.sh", - "time cd sub"]) + "time cd sub", '"source" "$ENV"', "'cd' \"$D\"", r'\eval "$X"', '"." env.sh', + 'c"d" "$D"', '"pushd" "$D"', 'builtin "cd" "$D"', '"cd" $(pwd)']) def test_a_pushd_eval_or_source_in_an_earlier_call_also_loses_the_directory(first): events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) assert _reads(events) == [] From e61e812edad4ae34392a991e59054992b087e66c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 2 Nov 2026 21:09:04 +0000 Subject: [PATCH 23/42] fix(repo): a directory builtin the parser does not follow moves it Settles the review points that `X="a b" cd "$D"` and `time -p cd sub` moved nothing. The walker matched the builtins on blank-split words for a part with an expansion, so an assignment holding a blank hid the `cd`, and it skipped `time`, `command` or `builtin` only right before a builtin's name, so an option between them (`time -p`, `command -p`, `builtin --`) hid it too. A later relative read in the same command or in the next Claude Code Bash call was then taken at the old directory. Rather than one more special case per shape, one conservative fallback: `_runs_dir` now reads a part's words with shlex where it can split them (so an assignment with a blank is skipped like any other), and a new `_may_run_dir` counts a part as moving the shell when a `builtin`, `command` or `time` the walker did not skip is followed by a directory builtin's name anywhere in the part. Text shlex cannot split is not understood at all, so any builtin's name in it counts. The forms the walker does follow are unchanged: `cd sub`, a backgrounded or piped one, a subshell. The failed-read rule is left as it was: a part before the read that is not surely a `cd` keeps the read on a non-zero exit, as its exit may be the one that failed. Its docstring says so. The contract's limits list states the rule and points at the drop-only forms tracked in #374. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 6 +++- dokimi/dokimi/repos/cli_traces.py | 35 +++++++++++++------ dokimi/tests/test_repo_cli_traces.py | 16 +++++++++ dokimi/tests/test_repo_events_bash.py | 20 +++++++++++ 5 files changed, 67 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index d4679c217..042c6cfea 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read). In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow that names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`) counts as moving the shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index b986ea461..67de7f034 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1188,7 +1188,11 @@ observations of that run. It cannot show: relative reads of both (this only drops reads; no real trace was checked); - a `cd` inside a shell function the agent defined and then called: the call is not followed, so a later relative read is taken from the directory - before it (an `eval`, `source` or `.` is taken as moving the shell); + before it (an `eval`, `source` or `.` is taken as moving the shell). A + command the parser does not fully follow that names a directory builtin + (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd`) counts as moving the + shell; the forms that only drop a read are listed in + [#374](https://github.com/intel/dokimi/issues/374); - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: they appear with `observed: unknown`. A skill body is loaded on demand, so it diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 5ae8e5106..dad67293a 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -53,6 +53,7 @@ # after these builtins the directory is not known: a directory stack, or a script that may run a ``cd`` _DIR_LOST = {"pushd", "popd", "eval", "source", "."} _DIRS = {"cd"} | _DIR_LOST # the builtins that change, or may change, the directory +_PREFIXES = {"builtin", "command", "time"} # each runs the directory builtin named after it in this shell _PIPE = re.compile(r"(? list[str]: relative path is joined to it and normalised (no ``.`` or ``..``). The directory is unknown (a relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``; none of these when backgrounded or - piped, as it then ran in a subshell: ``source env.sh & false || cat a`` reads a), ``env -C`` / ``--chdir`` (the + piped, as it then ran in a subshell: ``source env.sh & false || cat a`` reads a), a part the walker does not + follow that names one of these (``time -p cd sub``, ``X="a b" cd "$D"``: :func:`_may_run_dir`), ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records @@ -247,7 +249,7 @@ def command(self, words: list[str], after: str, chained: bool = False) -> list[s name = _run_name(words[0], via_env) if name is None: return [] - if name in _DIR_LOST or (conditional and name == "cd"): + if name in _DIR_LOST or (conditional and name == "cd") or _may_run_dir(words): self.lose(after != "&") elif name == "cd": self.gated |= chained @@ -293,13 +295,24 @@ def _cd(here: str | None, args: list[str]) -> str | None: def _runs_dir(outer: str) -> bool: - """Whether the outer text of a part (:class:`~dokimi.repos.shell_split.Part`) runs a directory builtin, as - ``cd $(...)`` does (a subshell ``(cd x; ...)`` has no such text, so it changes nothing outside itself).""" - words = outer.split() - while words and (words[0] in _KEYWORDS or words[0] == "{"): # a quoted ``"if"`` is no keyword: skipped unquoted + """Whether a part's text, or its outer text (:class:`~dokimi.repos.shell_split.Part`), runs a directory builtin, as + ``cd $(...)`` and ``X="a b" cd "$D"`` do, or may (:func:`_may_run_dir`); a subshell ``(cd x; ...)`` has no such + text, so it changes nothing outside itself. Text shlex cannot split is not understood: any builtin's name in it counts.""" + try: + words = shlex.split(outer) + except ValueError: + return any(w in _DIRS for w in _unquoted(outer.split())) + while words and (words[0] in _KEYWORDS or words[0] == "{"): words = words[1:] - words, via_env, _ = _command_words(_unquoted(words)) - return bool(words) and words[0] in _DIRS and not via_env + words, via_env, _ = _command_words(words) + return bool(words) and not via_env and (words[0] in _DIRS or _may_run_dir(words)) + + +def _may_run_dir(words: list[str]) -> bool: + """The rule for a part the walker does not fully follow: its unquoted ``words``, from the name on, start with a + ``builtin``, ``command`` or ``time`` that :func:`_command_words` did not skip, as an option comes first (``time -p + cd sub``, ``command -p cd``, ``builtin -- cd``), and a later word names a directory builtin: the shell may move.""" + return bool(words) and words[0] in _PREFIXES and any(w in _DIRS for w in words[1:]) def _in_own_shell(outer: str, after: str) -> bool: @@ -482,7 +495,7 @@ def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: while words: if _ASSIGN.match(words[0]): words = words[1:] - elif words[0] in ("builtin", "command", "time") and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program + elif words[0] in _PREFIXES and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program words = words[1:] elif _program(words[0]) == "env": words, via_env = words[1:], True @@ -630,7 +643,9 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read. Otherwise the exit code does not say which read failed (``cat a missing`` prints ``a`` and - exits 1), so the reads are kept: also when only one of the files is known (``cd -; cat missing /ws/a``). + exits 1), so the reads are kept: also when only one of the files is known (``cd -; cat missing /ws/a``). A part + before it that is not surely a ``cd`` (``X="a b" cd sub``, ``time -p cd sub``) keeps them too: its exit may be + the one that failed, and a read is dropped only when that file surely was not printed. """ if rc in (None, 0) or not reads or not all(_is_cd(part.text) for part in split_script(script)[:-1]): return False diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index a099bec59..fd5b4ec6e 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -218,6 +218,22 @@ def test_a_quoted_or_escaped_directory_builtin_loses_the_directory(command): assert shell_reads(command, "sub") == [] +@pytest.mark.parametrize("command", [ + 'X="a b" cd "$D"; cat AGENTS.md', "time -p cd sub; cat AGENTS.md", "command -p cd sub && cat AGENTS.md", + "builtin -- cd sub; cat AGENTS.md", 'time -p "cd" sub; cat AGENTS.md', 'X="a b" source "$E"; cat AGENTS.md', + "time -p pushd sub; cat AGENTS.md", "command -p cd $(pwd); cat AGENTS.md", +]) +def test_a_directory_builtin_the_walker_does_not_follow_loses_the_directory(command): + # an assignment with a blank before it, or an option after ``time`` / ``command`` / ``builtin``: it may move + assert shell_reads(command, "sub") == [] + + +@pytest.mark.parametrize("command", ["time -p cd sub & cat AGENTS.md", 'X="a b" cd "$D" | cat; cat AGENTS.md', + "time -p make; cat AGENTS.md", "command -v git; cat AGENTS.md"]) +def test_an_unfollowed_builtin_in_a_job_or_a_pipe_or_none_keeps_the_directory(command): + assert shell_reads(command, "sub") == ["sub/AGENTS.md"] + + @pytest.mark.parametrize("command", [ '"env" cd "$D"; cat AGENTS.md', "'\\cd' \"$D\"; cat AGENTS.md", '"./cd" "$D"; cat AGENTS.md', '"source" "$E" & cat AGENTS.md', '"cd" "$D" | cat; cat AGENTS.md', diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 1b31ae6b8..061df6834 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -174,6 +174,9 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ("{ cd $(pwd); }", True), ("{ { cd sub; }; }", True), ('"source" "$ENV"', True), ("'cd' \"$D\"", True), (r'\eval "$X"', True), ('"." env.sh', True), # quoted names ('"cd" "$D" &', False), ('"source" "$E" | cat', False), ('"env" cd "$D"', False), ("'\\cd' \"$D\"", False), + ('X="a b" cd "$D"', True), ("time -p cd sub", True), ("command -p cd sub", True), ("builtin -- cd sub", True), + ("time -p cd sub &", False), ('X="a b" cd "$D" | cat', False), ("time -p make", False), # not followed: may move + ('time -p cd "$D', True), # shlex cannot split it: any builtin's name counts ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @@ -213,6 +216,23 @@ def test_a_source_eval_or_computed_cd_in_a_job_or_a_pipeline_keeps_the_calls_lat assert _reads(events) == ["AGENTS.md", "README.md"] +_UNFOLLOWED = ['X="a b" cd "$D"', "time -p cd sub", "command -p cd sub", "builtin -- cd sub"] + + +@pytest.mark.parametrize("first", _UNFOLLOWED) +def test_a_directory_builtin_the_parser_does_not_follow_moves_the_shell(first): + # an assignment with a blank, or an option before the builtin: the shell may have moved, so no relative read + assert _reads(_events(_call("a", f"{first} && cat AGENTS.md"), _result("a"))) == [] + events = _events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b")) + assert _reads(events) == [] + + +def test_a_failed_read_after_a_part_not_surely_a_cd_keeps_its_read(): + # ``_is_cd`` splits at blanks: ``X="a b" cd sub`` is not surely a cd, so the exit may be its own + events = _events(_call("a", 'X="a b" cd sub && cat AGENTS.md'), _result("a", "error", True)) + assert [(e["kind"], e.get("path"), e["rc"]) for e in events] == [("read", "sub/AGENTS.md", 0), ("command", None, 1)] + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From 33634a8587041b841a7aaaee60caf65cd22a2167 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:09:21 +0000 Subject: [PATCH 24/42] fix(repo): only the command word can name a directory builtin Settles the review point that the fallback for parts the parser does not fully follow fired on a builtin's name anywhere in the part. `time find . -name x`, `time git add .`, `command -v cd` or `builtin printf '%s' .` marked the shell as moved, so every later relative Bash read of the trace was dropped. `_may_run_dir` and `_runs_dir` now look at the command word only: past `builtin`, `command` and `time` with their options (`time -p`, `command -p`, `--`), the next word moves the shell only if it is a directory builtin. `command -v` / `-V` describes a command and runs nothing. For text shlex cannot split, the words are unquoted one by one and the same command word counts; when a quote up to it does not close (`X="a b c" cd "$D`), any builtin's name in the part still counts. `time -p cd sub`, `command -p cd sub` and `builtin -- cd sub` still move. The contract's limits list and the changelog fragment state the narrowed rule. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 7 ++- dokimi/dokimi/repos/cli_traces.py | 62 +++++++++++++------ dokimi/tests/test_repo_cli_traces.py | 4 +- dokimi/tests/test_repo_events_bash.py | 18 +++++- 5 files changed, 69 insertions(+), 24 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 042c6cfea..2df311b25 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow that names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`) counts as moving the shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 67de7f034..3ee23a515 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1189,9 +1189,10 @@ observations of that run. It cannot show: - a `cd` inside a shell function the agent defined and then called: the call is not followed, so a later relative read is taken from the directory before it (an `eval`, `source` or `.` is taken as moving the shell). A - command the parser does not fully follow that names a directory builtin - (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd`) counts as moving the - shell; the forms that only drop a read are listed in + command the parser does not fully follow counts as moving the shell when + its command word, after `builtin`, `command` or `time` and their options, + names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`; not + `time git add .`); the forms that only drop a read are listed in [#374](https://github.com/intel/dokimi/issues/374); - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index dad67293a..7197353a0 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -91,7 +91,7 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``; none of these when backgrounded or piped, as it then ran in a subshell: ``source env.sh & false || cat a`` reads a), a part the walker does not - follow that names one of these (``time -p cd sub``, ``X="a b" cd "$D"``: :func:`_may_run_dir`), ``env -C`` / ``--chdir`` (the + follow whose command word is one of these (``time -p cd sub``, ``X="a b" cd "$D"``: :func:`_may_run_dir`), ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records @@ -297,22 +297,46 @@ def _cd(here: str | None, args: list[str]) -> str | None: def _runs_dir(outer: str) -> bool: """Whether a part's text, or its outer text (:class:`~dokimi.repos.shell_split.Part`), runs a directory builtin, as ``cd $(...)`` and ``X="a b" cd "$D"`` do, or may (:func:`_may_run_dir`); a subshell ``(cd x; ...)`` has no such - text, so it changes nothing outside itself. Text shlex cannot split is not understood: any builtin's name in it counts.""" + text, so it changes nothing outside itself. Text shlex cannot split is read word by word (:func:`_unquoted`): its + command word counts, and when a quote up to it does not close (``X="a b c" cd``), any builtin's name in it counts.""" try: - words = shlex.split(outer) + words, told = shlex.split(outer), None except ValueError: - return any(w in _DIRS for w in _unquoted(outer.split())) - while words and (words[0] in _KEYWORDS or words[0] == "{"): - words = words[1:] - words, via_env, _ = _command_words(words) - return bool(words) and not via_env and (words[0] in _DIRS or _may_run_dir(words)) + raw = outer.split() + words, told = _unquoted(raw), [_unquote(w) is not None for w in raw] + rest = words + while rest and (rest[0] in _KEYWORDS or rest[0] == "{"): + rest = rest[1:] + rest, via_env, _ = _command_words(rest) + at = _command_at(rest) + if at is None or at >= len(rest): + return False + if told is not None and not all(told[:len(words) - len(rest) + at + 1]): + return any(w in _DIRS for w in words) + return not via_env and rest[at] in _DIRS def _may_run_dir(words: list[str]) -> bool: """The rule for a part the walker does not fully follow: its unquoted ``words``, from the name on, start with a ``builtin``, ``command`` or ``time`` that :func:`_command_words` did not skip, as an option comes first (``time -p - cd sub``, ``command -p cd``, ``builtin -- cd``), and a later word names a directory builtin: the shell may move.""" - return bool(words) and words[0] in _PREFIXES and any(w in _DIRS for w in words[1:]) + cd sub``, ``command -p cd``, ``builtin -- cd``), and the command word after it names a directory builtin.""" + at = _command_at(words) if words[:1] and words[0] in _PREFIXES else None + return at is not None and at < len(words) and words[at] in _DIRS + + +def _command_at(words: list[str]) -> int | None: + """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and ``time`` with their options + (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only describes the command.""" + at = 0 + while at < len(words) and words[at] in _PREFIXES: + prefix, at = words[at], at + 1 + while at < len(words) and words[at].startswith("-") and words[at] != "-": + if prefix == "command" and set(words[at]) & {"v", "V"}: + return None + at += 1 + if words[at - 1] == "--": + break + return at def _in_own_shell(outer: str, after: str) -> bool: @@ -476,14 +500,16 @@ def _unquoted(words: list[str]) -> list[str]: """Words split at blanks, each with its quotes and backslash escapes removed and nothing expanded (``"cd"``, ``'cd'``, ``\\cd`` and ``c"d"`` are ``cd``; ``"$D"`` is ``$D``), as ``shlex.split`` gives them; a word whose quotes do not close here (``"a b"`` split at its blank) stays as it is.""" - out = [] - for word in words: - try: - split = shlex.split(word) - except ValueError: - split = [] - out.append(split[0] if len(split) == 1 else word) - return out + return [w if (u := _unquote(w)) is None else u for w in words] + + +def _unquote(word: str) -> str | None: + """``word`` with its quotes and escapes removed, as :func:`_unquoted` does; None when its quote does not close.""" + try: + split = shlex.split(word) + except ValueError: + return None + return split[0] if len(split) == 1 else None def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fd5b4ec6e..f8ab079c6 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -229,7 +229,9 @@ def test_a_directory_builtin_the_walker_does_not_follow_loses_the_directory(comm @pytest.mark.parametrize("command", ["time -p cd sub & cat AGENTS.md", 'X="a b" cd "$D" | cat; cat AGENTS.md', - "time -p make; cat AGENTS.md", "command -v git; cat AGENTS.md"]) + "time -p make; cat AGENTS.md", "command -v git; cat AGENTS.md", + "time git add .; cat AGENTS.md", "command -v cd; cat AGENTS.md", + "builtin printf '%s' . && cat AGENTS.md", "time find . -name x; cat AGENTS.md"]) def test_an_unfollowed_builtin_in_a_job_or_a_pipe_or_none_keeps_the_directory(command): assert shell_reads(command, "sub") == ["sub/AGENTS.md"] diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 061df6834..37e163496 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -176,7 +176,7 @@ def test_a_call_that_moves_no_shell_keeps_the_next_relative_read(first): ('"cd" "$D" &', False), ('"source" "$E" | cat', False), ('"env" cd "$D"', False), ("'\\cd' \"$D\"", False), ('X="a b" cd "$D"', True), ("time -p cd sub", True), ("command -p cd sub", True), ("builtin -- cd sub", True), ("time -p cd sub &", False), ('X="a b" cd "$D" | cat', False), ("time -p make", False), # not followed: may move - ('time -p cd "$D', True), # shlex cannot split it: any builtin's name counts + ('time -p cd "$D', True), ('X="a b c" cd "$D', True), # shlex cannot split it: the command word, else any word ]) def test_only_a_directory_builtin_in_the_calls_own_shell_moves_it(command, moves): assert shell_reads_moved(command)[1] is moves @@ -233,6 +233,22 @@ def test_a_failed_read_after_a_part_not_surely_a_cd_keeps_its_read(): assert [(e["kind"], e.get("path"), e["rc"]) for e in events] == [("read", "sub/AGENTS.md", 0), ("command", None, 1)] +@pytest.mark.parametrize("first", ["time find . -name x", "time cat AGENTS.md .", "time git add .", "time make -C .", + "command -v cd", "command -V cd", "builtin printf '%s' ."]) +def test_a_builtins_name_as_an_argument_moves_no_shell(first): + # only the command word, past ``time`` / ``command`` / ``builtin`` and their options, names the builtin + assert shell_reads_moved(first)[1] is False + assert _reads(_events(_call("a", f"{first}; cat AGENTS.md"), _result("a"))) == ["AGENTS.md"] + assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == ["AGENTS.md"] + + +def test_a_builtins_name_after_a_quote_shlex_cannot_split_moves_no_shell(): + # the command word is told: grep (the splitter does not know ``$'...'``, so only the next call is checked) + first = r"grep $'it\'s' ." + assert shell_reads_moved(first)[1] is False + assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == ["AGENTS.md"] + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From 1e5ccd765073c6df64dfb87f509d3b3c414f1252 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:11:12 +0000 Subject: [PATCH 25/42] refactor(repo): move a shell command's word helpers to shell_words cli_traces.py is at its 800-line cap, and the next fixes to how the walker finds a part's command word need room. The helpers that find the command word past `NAME=value`, `builtin`, `command`, `time` and `env`, and that tell whether it runs a directory builtin (`_command_words`, `_command_at`, `_runs_dir`, `_may_run_dir`, `_is_cd`, `_run_name`, `_cd`, `_unquoted` and their constants), move as they are to a new module beside it, `repos/shell_words.py`. No code changes; architecture.md names the module. Signed-off-by: Anton Volkov --- dokimi/docs/architecture.md | 4 +- dokimi/dokimi/repos/cli_traces.py | 129 +------------------------- dokimi/dokimi/repos/shell_words.py | 140 +++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 129 deletions(-) create mode 100644 dokimi/dokimi/repos/shell_words.py diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 52994397f..18dcd83c8 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -316,7 +316,9 @@ next release. - `dokimi/repos/events.py` — converts a harness trace to the `events.jsonl` run trace; `dokimi/repos/cli_traces.py` holds the Codex and Copilot JSON converters and the parser of the files a shell command prints - (`dokimi/repos/shell_split.py` splits the command into its parts), + (`dokimi/repos/shell_split.py` splits the command into its parts, and + `dokimi/repos/shell_words.py` finds a part's command word and the + directory builtins it runs), and `dokimi/repos/answers.py` the `answer_format` conversions that turn a native review answer into a findings block. - `dokimi/repos/l0/` — `repo scan` (L0): checks the links, paths, diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 7197353a0..01622af47 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -27,6 +27,7 @@ from typing import Any from .shell_split import split_script +from .shell_words import _DIR_LOST, _KEYWORDS, _cd, _command_words, _is_cd, _may_run_dir, _run_name, _runs_dir from .transcript import Transcript, _parse_time, clip_output, relative_path _BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) @@ -47,13 +48,6 @@ _STDERR_FILE = re.compile(r"(?>?[ \t]*[^\s;&|<>()]+") # redirects apply in order: after ``2>/dev/null`` or ``2>file``, a ``1>&2`` sends stdout there too, away from the agent _STDOUT_AFTER_STDERR = re.compile(r"(?>?(?!&)[ \t]*[^\s;&|<>()]+.*?(?&2(?!\d)") -_SYSTEM_BIN = {"/bin", "/usr/bin", "/usr/local/bin"} -_ASSIGN = re.compile(r"[A-Za-z_]\w*=") # a NAME=value word: a variable set for the command that follows -_KEYWORDS = {"if", "then", "else", "elif", "do", "while", "until", "!"} -# after these builtins the directory is not known: a directory stack, or a script that may run a ``cd`` -_DIR_LOST = {"pushd", "popd", "eval", "source", "."} -_DIRS = {"cd"} | _DIR_LOST # the builtins that change, or may change, the directory -_PREFIXES = {"builtin", "command", "time"} # each runs the directory builtin named after it in this shell _PIPE = re.compile(r"(? str: return posixpath.normpath(f if f.startswith("/") else posixpath.join(self.here or "", f)) -def _run_name(name: str, via_env: bool) -> str | None: - """The name a command runs as, or None when it is a directory builtin (``cd``, ``pushd``, ``eval``, ...) that does not run.""" - if via_env and (name == "cd" or name in _DIR_LOST): # ``env cd x`` runs no builtin: nothing moves - return None - if "/" in name: # ``/bin/cat`` is cat; ``./cat`` or ``/bin/cd`` is some other program - name = posixpath.basename(name) if posixpath.dirname(name) in _SYSTEM_BIN else "" - if name == "cd" or name in _DIR_LOST: - return None - return name - - -def _cd(here: str | None, args: list[str]) -> str | None: - """The directory after ``cd `` from ``here`` (``None``: unknown).""" - rest = list(args) - while rest and rest[0] in ("-L", "-P", "-e", "-@"): - rest.pop(0) - if rest and rest[0] == "--": - rest.pop(0) - if len(rest) != 1 or rest[0] == "-" or rest[0].startswith("~"): - return None # home, the previous directory, or something we do not track - target = rest[0] - if target.startswith("/"): - return posixpath.normpath(target) - return None if here is None else posixpath.normpath(posixpath.join(here, target)) - - -def _runs_dir(outer: str) -> bool: - """Whether a part's text, or its outer text (:class:`~dokimi.repos.shell_split.Part`), runs a directory builtin, as - ``cd $(...)`` and ``X="a b" cd "$D"`` do, or may (:func:`_may_run_dir`); a subshell ``(cd x; ...)`` has no such - text, so it changes nothing outside itself. Text shlex cannot split is read word by word (:func:`_unquoted`): its - command word counts, and when a quote up to it does not close (``X="a b c" cd``), any builtin's name in it counts.""" - try: - words, told = shlex.split(outer), None - except ValueError: - raw = outer.split() - words, told = _unquoted(raw), [_unquote(w) is not None for w in raw] - rest = words - while rest and (rest[0] in _KEYWORDS or rest[0] == "{"): - rest = rest[1:] - rest, via_env, _ = _command_words(rest) - at = _command_at(rest) - if at is None or at >= len(rest): - return False - if told is not None and not all(told[:len(words) - len(rest) + at + 1]): - return any(w in _DIRS for w in words) - return not via_env and rest[at] in _DIRS - - -def _may_run_dir(words: list[str]) -> bool: - """The rule for a part the walker does not fully follow: its unquoted ``words``, from the name on, start with a - ``builtin``, ``command`` or ``time`` that :func:`_command_words` did not skip, as an option comes first (``time -p - cd sub``, ``command -p cd``, ``builtin -- cd``), and the command word after it names a directory builtin.""" - at = _command_at(words) if words[:1] and words[0] in _PREFIXES else None - return at is not None and at < len(words) and words[at] in _DIRS - - -def _command_at(words: list[str]) -> int | None: - """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and ``time`` with their options - (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only describes the command.""" - at = 0 - while at < len(words) and words[at] in _PREFIXES: - prefix, at = words[at], at + 1 - while at < len(words) and words[at].startswith("-") and words[at] != "-": - if prefix == "command" and set(words[at]) & {"v", "V"}: - return None - at += 1 - if words[at - 1] == "--": - break - return at - - def _in_own_shell(outer: str, after: str) -> bool: """Whether the part with this outer text runs in the script's shell: not backgrounded by ``after``, nor a pipeline stage (a pipe in quotes or escaped, ``\\|``, is not one).""" @@ -490,50 +413,6 @@ def _outside_quotes(part: str, rx: re.Pattern[str]) -> str: return "".join(out) + part[last:] -def _program(word: str) -> str: - """The program a word names when it is a bare name or a system path (``env``, ``/usr/bin/env``); ``""`` for a - path elsewhere (``./env``), which is some program of the repository, not the one the system has.""" - return posixpath.basename(word) if posixpath.dirname(word) in ("", *_SYSTEM_BIN) else "" - - -def _unquoted(words: list[str]) -> list[str]: - """Words split at blanks, each with its quotes and backslash escapes removed and nothing expanded (``"cd"``, - ``'cd'``, ``\\cd`` and ``c"d"`` are ``cd``; ``"$D"`` is ``$D``), as ``shlex.split`` gives them; a word - whose quotes do not close here (``"a b"`` split at its blank) stays as it is.""" - return [w if (u := _unquote(w)) is None else u for w in words] - - -def _unquote(word: str) -> str | None: - """``word`` with its quotes and escapes removed, as :func:`_unquoted` does; None when its quote does not close.""" - try: - split = shlex.split(word) - except ValueError: - return None - return split[0] if len(split) == 1 else None - - -def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: - """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes - directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or ``time`` before a - directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped. - ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" - via_env = chdir = False - while words: - if _ASSIGN.match(words[0]): - words = words[1:] - elif words[0] in _PREFIXES and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program - words = words[1:] - elif _program(words[0]) == "env": - words, via_env = words[1:], True - while words and (words[0].startswith("-") or _ASSIGN.match(words[0])): - opt = words[0] - chdir |= opt.startswith(("-C", "--chdir")) - words = words[2:] if opt in ("-C", "-u", "--chdir", "--unset") else words[1:] # these take the next word as a value - else: - break - return words, via_env, chdir - - def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: out = Transcript() messages: list[str] = [] @@ -678,12 +557,6 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: return _walk_script(script, "")[1].operands == 1 # the printer's own operands, not the reads kept of them -def _is_cd(part: str) -> bool: - """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``, ``"cd"``; not ``env cd``).""" - words, via_env, _ = _command_words(_unquoted(part.split())) - return words[:1] == ["cd"] and not via_env - - class _CopilotRun: """What ``copilot_json`` collects across the lines: the first timestamp, tool events by call id, the answer (the final one, else the last message) and the turn count.""" diff --git a/dokimi/dokimi/repos/shell_words.py b/dokimi/dokimi/repos/shell_words.py new file mode 100644 index 000000000..9b01640b8 --- /dev/null +++ b/dokimi/dokimi/repos/shell_words.py @@ -0,0 +1,140 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A shell command's words, for :func:`~dokimi.repos.cli_traces.shell_reads`: the command word past what comes before +it (``NAME=value``, ``builtin``, ``command``, ``time``, ``env``), and whether it runs a directory builtin.""" + +from __future__ import annotations + +import posixpath +import re +import shlex + +_SYSTEM_BIN = {"/bin", "/usr/bin", "/usr/local/bin"} +_ASSIGN = re.compile(r"[A-Za-z_]\w*=") # a NAME=value word: a variable set for the command that follows +_KEYWORDS = {"if", "then", "else", "elif", "do", "while", "until", "!"} +# after these builtins the directory is not known: a directory stack, or a script that may run a ``cd`` +_DIR_LOST = {"pushd", "popd", "eval", "source", "."} +_DIRS = {"cd"} | _DIR_LOST # the builtins that change, or may change, the directory +_PREFIXES = {"builtin", "command", "time"} # each runs the directory builtin named after it in this shell + + +def _run_name(name: str, via_env: bool) -> str | None: + """The name a command runs as, or None when it is a directory builtin (``cd``, ``pushd``, ``eval``, ...) that does not run.""" + if via_env and (name == "cd" or name in _DIR_LOST): # ``env cd x`` runs no builtin: nothing moves + return None + if "/" in name: # ``/bin/cat`` is cat; ``./cat`` or ``/bin/cd`` is some other program + name = posixpath.basename(name) if posixpath.dirname(name) in _SYSTEM_BIN else "" + if name == "cd" or name in _DIR_LOST: + return None + return name + + +def _cd(here: str | None, args: list[str]) -> str | None: + """The directory after ``cd `` from ``here`` (``None``: unknown).""" + rest = list(args) + while rest and rest[0] in ("-L", "-P", "-e", "-@"): + rest.pop(0) + if rest and rest[0] == "--": + rest.pop(0) + if len(rest) != 1 or rest[0] == "-" or rest[0].startswith("~"): + return None # home, the previous directory, or something we do not track + target = rest[0] + if target.startswith("/"): + return posixpath.normpath(target) + return None if here is None else posixpath.normpath(posixpath.join(here, target)) + + +def _runs_dir(outer: str) -> bool: + """Whether a part's text, or its outer text (:class:`~dokimi.repos.shell_split.Part`), runs a directory builtin, as + ``cd $(...)`` and ``X="a b" cd "$D"`` do, or may (:func:`_may_run_dir`); a subshell ``(cd x; ...)`` has no such + text, so it changes nothing outside itself. Text shlex cannot split is read word by word (:func:`_unquoted`): its + command word counts, and when a quote up to it does not close (``X="a b c" cd``), any builtin's name in it counts.""" + try: + words, told = shlex.split(outer), None + except ValueError: + raw = outer.split() + words, told = _unquoted(raw), [_unquote(w) is not None for w in raw] + rest = words + while rest and (rest[0] in _KEYWORDS or rest[0] == "{"): + rest = rest[1:] + rest, via_env, _ = _command_words(rest) + at = _command_at(rest) + if at is None or at >= len(rest): + return False + if told is not None and not all(told[:len(words) - len(rest) + at + 1]): + return any(w in _DIRS for w in words) + return not via_env and rest[at] in _DIRS + + +def _may_run_dir(words: list[str]) -> bool: + """The rule for a part the walker does not fully follow: its unquoted ``words``, from the name on, start with a + ``builtin``, ``command`` or ``time`` that :func:`_command_words` did not skip, as an option comes first (``time -p + cd sub``, ``command -p cd``, ``builtin -- cd``), and the command word after it names a directory builtin.""" + at = _command_at(words) if words[:1] and words[0] in _PREFIXES else None + return at is not None and at < len(words) and words[at] in _DIRS + + +def _command_at(words: list[str]) -> int | None: + """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and ``time`` with their options + (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only describes the command.""" + at = 0 + while at < len(words) and words[at] in _PREFIXES: + prefix, at = words[at], at + 1 + while at < len(words) and words[at].startswith("-") and words[at] != "-": + if prefix == "command" and set(words[at]) & {"v", "V"}: + return None + at += 1 + if words[at - 1] == "--": + break + return at + + +def _program(word: str) -> str: + """The program a word names when it is a bare name or a system path (``env``, ``/usr/bin/env``); ``""`` for a + path elsewhere (``./env``), which is some program of the repository, not the one the system has.""" + return posixpath.basename(word) if posixpath.dirname(word) in ("", *_SYSTEM_BIN) else "" + + +def _unquoted(words: list[str]) -> list[str]: + """Words split at blanks, each with its quotes and backslash escapes removed and nothing expanded (``"cd"``, + ``'cd'``, ``\\cd`` and ``c"d"`` are ``cd``; ``"$D"`` is ``$D``), as ``shlex.split`` gives them; a word + whose quotes do not close here (``"a b"`` split at its blank) stays as it is.""" + return [w if (u := _unquote(w)) is None else u for w in words] + + +def _unquote(word: str) -> str | None: + """``word`` with its quotes and escapes removed, as :func:`_unquoted` does; None when its quote does not close.""" + try: + split = shlex.split(word) + except ValueError: + return None + return split[0] if len(split) == 1 else None + + +def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: + """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes + directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or ``time`` before a + directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped. + ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" + via_env = chdir = False + while words: + if _ASSIGN.match(words[0]): + words = words[1:] + elif words[0] in _PREFIXES and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program + words = words[1:] + elif _program(words[0]) == "env": + words, via_env = words[1:], True + while words and (words[0].startswith("-") or _ASSIGN.match(words[0])): + opt = words[0] + chdir |= opt.startswith(("-C", "--chdir")) + words = words[2:] if opt in ("-C", "-u", "--chdir", "--unset") else words[1:] # these take the next word as a value + else: + break + return words, via_env, chdir + + +def _is_cd(part: str) -> bool: + """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``, ``"cd"``; not ``env cd``).""" + words, via_env, _ = _command_words(_unquoted(part.split())) + return words[:1] == ["cd"] and not via_env From f7f4f657b437cb9933f59dda6c726f1651d82878 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:13:34 +0000 Subject: [PATCH 26/42] fix(repo): time is the keyword only as the first unquoted word Settles the review point that `X=1 time cd sub; cat a` and `\time cd sub; cat a` read `sub/a` and moved the shell. Bash takes `time` as its keyword only as the first word of a command, unquoted; after an assignment, or quoted or escaped, it is /usr/bin/time, which runs the `cd` in a child process, so the shell stays and `cat a` reads `a`. The walker now skips `time` before a directory builtin, and as a prefix with options, only when the part's raw text starts with the keyword (`_time_keyword`); `_is_cd` follows it, so a failed `X=1 time cd sub && cat AGENTS.md` keeps its read like any part that is not surely a `cd`. The same point found the reverse in `case x in a) cd sub;; esac`, which moved nothing. A `case ... esac` is a compound the walker does not follow: from its `case` part to its `esac`, a part naming a directory builtin counts as moving the shell, and the reads in it are not kept, as an arm may or may not run. Later arms and arms on their own lines count too. The contract's limits list and the changelog fragment say both. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 +- dokimi/dokimi/repos/cli_traces.py | 34 +++++++++--- dokimi/dokimi/repos/shell_words.py | 54 ++++++++++++------- dokimi/tests/test_repo_events_bash.py | 22 +++++++- 5 files changed, 86 insertions(+), 30 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 2df311b25..335e18d00 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 3ee23a515..a34a48be5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1192,7 +1192,9 @@ observations of that run. It cannot show: command the parser does not fully follow counts as moving the shell when its command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`; not - `time git add .`); the forms that only drop a read are listed in + `time git add .`), and so does one in a `case` arm. `time` is that + keyword only as the first word, unquoted (`X=1 time cd sub` and + `\time cd sub` run /usr/bin/time); the forms that only drop a read are listed in [#374](https://github.com/intel/dokimi/issues/374); - when a source entered the request, or in what order (no timestamps); - start-up imports and skill listings from budget, which reach does not probe: diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 01622af47..ab6d92d83 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -27,7 +27,8 @@ from typing import Any from .shell_split import split_script -from .shell_words import _DIR_LOST, _KEYWORDS, _cd, _command_words, _is_cd, _may_run_dir, _run_name, _runs_dir +from .shell_words import (_DIR_LOST, _DIRS, _KEYWORDS, _cd, _command_words, _is_cd, _may_run_dir, _run_name, _runs_dir, + _time_keyword, _unquoted) from .transcript import Transcript, _parse_time, clip_output, relative_path _BASH_WRAP = re.compile(r"^(?:/\S*/)?(?:ba|z)?sh -lc (.+)$", re.S) @@ -85,7 +86,8 @@ def shell_reads(script: str, cwd: str = "") -> list[str]: relative read is not recorded) after ``cd -``, a bare ``cd``, a ``cd`` to a computed path, a ``pushd`` / ``popd``, an ``eval``, ``source`` or ``.`` (it may run a ``cd``; none of these when backgrounded or piped, as it then ran in a subshell: ``source env.sh & false || cat a`` reads a), a part the walker does not - follow whose command word is one of these (``time -p cd sub``, ``X="a b" cd "$D"``: :func:`_may_run_dir`), ``env -C`` / ``--chdir`` (the + follow whose command word is one of these (``time -p cd sub``, ``X="a b" cd "$D"``: :func:`_may_run_dir`), a ``case`` + arm that names one (``case`` is not followed: its reads are not kept either), ``env -C`` / ``--chdir`` (the command's reads, but absolute ones), a ``cd`` after ``if`` / ``then`` / ``do`` / ``else`` (it may not run), a ``cd`` after ``&&`` / ``||`` once the chain holding it ends with ``;``, ``&`` or a newline (it may have been skipped), or a glob or brace expansion (the files are not known). A subshell ``( ... )`` or `` `...` `` records @@ -121,7 +123,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: if not chained and walk.gated: # a cd after && or || may not have run: the next chain starts in no known directory walk.here, walk.gated = None, False walk.chain(op, chained) - if walk.in_function(_bare(part)): # a definition runs nothing + if walk.in_function(_bare(part)) or walk.in_case(outer): # a definition runs nothing; a case is not followed continue if op == "||" and walk.moved: walk.here = None @@ -132,7 +134,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: continue words = walk.words(part, after) if words: - out.extend(walk.command(words, after, chained)) + out.extend(walk.command(words, after, chained, _time_keyword(part))) walk.chain(previous, False) # a list backgrounded at the end (``cd sub && make &``) ran in a subshell too return out, walk @@ -155,6 +157,7 @@ def __init__(self, here: str | None) -> None: self.list_start: _Spot = (here, False, False) # the spot where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition + self.case = 0 # the depth inside ``case ... esac`` self.operands: int | None = None # how many files the last printing command named def spot(self) -> _Spot: @@ -182,6 +185,20 @@ def in_function(self, bare: str) -> bool: self.function = max(self.function + len(_GROUP_OPEN.findall(bare)) - len(_GROUP_CLOSE.findall(bare)), 0) return True + def in_case(self, outer: str) -> bool: + """Whether ``outer`` is part of a ``case ... esac``, which is not followed: an arm may run or not, so one naming a + directory builtin (as any word: ``case x in a) cd sub;; esac``) may move the shell, and its reads are not kept.""" + words = _unquoted(outer.split()) + while words[:1] and (words[0] in _KEYWORDS or words[0] == "{"): + words = words[1:] + self.case += words[:1] == ["case"] + if not self.case: + return False + if any(w in _DIRS for w in words): + self.lose() + self.case -= words[:1] == ["esac"] + return True + def lose(self, own_shell: bool = True) -> None: """A directory builtin went where it is not known. Not ``own_shell`` (backgrounded, or a pipeline stage): it ran in a subshell, so this shell's directory stays known (``source env.sh & false || cat a`` reads a).""" @@ -231,19 +248,20 @@ def words(self, part: str, after: str = "") -> list[str]: words = words[1:] return words - def command(self, words: list[str], after: str, chained: bool = False) -> list[str]: + def command(self, words: list[str], after: str, chained: bool = False, timed: bool = False) -> list[str]: """Move the directory for a ``cd``; the files a printing command reads. ``chained``: the command follows - ``&&`` or ``||``, so a ``cd`` in it runs only if the command before it succeeded.""" + ``&&`` or ``||``, so a ``cd`` in it runs only if the command before it succeeded; ``timed``: it starts with + bash's ``time`` keyword (:func:`~dokimi.repos.shell_words._time_keyword`).""" conditional = False while words and words[0] in _KEYWORDS: # ``then cd x``: the cd may not run words, conditional = words[1:], True - words, via_env, chdir = _command_words(words) + words, via_env, chdir = _command_words(words, timed) if not words: return [] name = _run_name(words[0], via_env) if name is None: return [] - if name in _DIR_LOST or (conditional and name == "cd") or _may_run_dir(words): + if name in _DIR_LOST or (conditional and name == "cd") or _may_run_dir(words, timed): self.lose(after != "&") elif name == "cd": self.gated |= chained diff --git a/dokimi/dokimi/repos/shell_words.py b/dokimi/dokimi/repos/shell_words.py index 9b01640b8..667d92c64 100644 --- a/dokimi/dokimi/repos/shell_words.py +++ b/dokimi/dokimi/repos/shell_words.py @@ -17,6 +17,13 @@ _DIR_LOST = {"pushd", "popd", "eval", "source", "."} _DIRS = {"cd"} | _DIR_LOST # the builtins that change, or may change, the directory _PREFIXES = {"builtin", "command", "time"} # each runs the directory builtin named after it in this shell +# ``time`` is bash's keyword only as the first word, unquoted; else (``X=1 time``, ``\time``) it is /usr/bin/time, a child +_TIME = re.compile(r"\s*(?:(?:[{!]|if|then|else|elif|do|while|until)\s+)*time(?=\s|$)") + + +def _time_keyword(text: str) -> bool: + """Whether the command in ``text`` starts with bash's ``time`` keyword, which runs the rest in this shell.""" + return _TIME.match(text) is not None def _run_name(name: str, via_env: bool) -> str | None: @@ -55,11 +62,11 @@ def _runs_dir(outer: str) -> bool: except ValueError: raw = outer.split() words, told = _unquoted(raw), [_unquote(w) is not None for w in raw] - rest = words + rest, timed = words, _time_keyword(outer) while rest and (rest[0] in _KEYWORDS or rest[0] == "{"): rest = rest[1:] - rest, via_env, _ = _command_words(rest) - at = _command_at(rest) + rest, via_env, _ = _command_words(rest, timed) + at = _command_at(rest, timed) if at is None or at >= len(rest): return False if told is not None and not all(told[:len(words) - len(rest) + at + 1]): @@ -67,19 +74,21 @@ def _runs_dir(outer: str) -> bool: return not via_env and rest[at] in _DIRS -def _may_run_dir(words: list[str]) -> bool: +def _may_run_dir(words: list[str], timed: bool) -> bool: """The rule for a part the walker does not fully follow: its unquoted ``words``, from the name on, start with a - ``builtin``, ``command`` or ``time`` that :func:`_command_words` did not skip, as an option comes first (``time -p - cd sub``, ``command -p cd``, ``builtin -- cd``), and the command word after it names a directory builtin.""" - at = _command_at(words) if words[:1] and words[0] in _PREFIXES else None - return at is not None and at < len(words) and words[at] in _DIRS + ``builtin``, ``command`` or ``time`` (``timed``: the keyword) that :func:`_command_words` did not skip, as an option + comes first (``time -p cd sub``, ``command -p cd``, ``builtin -- cd``), and the command word after it names a + directory builtin.""" + at = _command_at(words, timed) + return at is not None and 0 < at < len(words) and words[at] in _DIRS -def _command_at(words: list[str]) -> int | None: - """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and ``time`` with their options - (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only describes the command.""" +def _command_at(words: list[str], timed: bool) -> int | None: + """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and, when ``timed``, a first + ``time``, with their options (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only + describes the command.""" at = 0 - while at < len(words) and words[at] in _PREFIXES: + while at < len(words) and _prefix(words[at], timed and at == 0): prefix, at = words[at], at + 1 while at < len(words) and words[at].startswith("-") and words[at] != "-": if prefix == "command" and set(words[at]) & {"v", "V"}: @@ -112,16 +121,21 @@ def _unquote(word: str) -> str | None: return split[0] if len(split) == 1 else None -def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: +def _prefix(word: str, timed: bool) -> bool: + """Whether ``word`` is ``builtin``, ``command``, or ``time`` as bash's keyword (``timed``), which run what follows.""" + return word in _PREFIXES and (word != "time" or timed) + + +def _command_words(words: list[str], timed: bool) -> tuple[list[str], bool, bool]: """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes - directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or ``time`` before a - directory builtin (which runs it), and ``env`` with its options and assignments come before the name and are skipped. - ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" + directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or a first ``time`` (``timed``: the + keyword) before a directory builtin (which runs it), and ``env`` with its options and assignments come before the + name and are skipped. ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" via_env = chdir = False while words: if _ASSIGN.match(words[0]): words = words[1:] - elif words[0] in _PREFIXES and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program + elif _prefix(words[0], timed) and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program words = words[1:] elif _program(words[0]) == "env": words, via_env = words[1:], True @@ -131,10 +145,12 @@ def _command_words(words: list[str]) -> tuple[list[str], bool, bool]: words = words[2:] if opt in ("-C", "-u", "--chdir", "--unset") else words[1:] # these take the next word as a value else: break + timed = False # past the first word, ``time`` is a program return words, via_env, chdir def _is_cd(part: str) -> bool: - """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``, ``"cd"``; not ``env cd``).""" - words, via_env, _ = _command_words(_unquoted(part.split())) + """Whether ``part`` runs the ``cd`` builtin, past what the walker skips (``time cd``, ``X=1 cd``, ``"cd"``; not + ``env cd``, nor ``X=1 time cd`` or ``\\time cd``, which run /usr/bin/time).""" + words, via_env, _ = _command_words(_unquoted(part.split()), _time_keyword(part)) return words[:1] == ["cd"] and not via_env diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 37e163496..cadf4c1f0 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -68,7 +68,8 @@ def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): @pytest.mark.parametrize("command", ["cat AGENTS.md && cd sub", "make && cat AGENTS.md", "(cd sub) && cat AGENTS.md", - "env cd sub && cat AGENTS.md"]) # ``env cd`` runs no builtin: some program failed + "env cd sub && cat AGENTS.md", # ``env cd`` runs no builtin: some program failed + "X=1 time cd sub && cat AGENTS.md", r"\time cd sub && cat AGENTS.md"]) # /usr/bin/time def test_a_failed_list_whose_other_part_is_not_a_cd_keeps_the_read(command): # the failure may be the other part's: the file may have been shown events = _events(_call("a", command), _result("a", "error", True)) @@ -249,6 +250,25 @@ def test_a_builtins_name_after_a_quote_shlex_cannot_split_moves_no_shell(): assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == ["AGENTS.md"] +@pytest.mark.parametrize("first", ["X=1 time cd sub", r"\time cd sub", '"time" cd sub', "X=1 time -p cd sub"]) +def test_a_time_that_is_not_the_keyword_runs_a_child_and_moves_no_shell(first): + # after an assignment, or quoted or escaped, ``time`` is /usr/bin/time: the cd runs in a child process + assert shell_reads_moved(f"{first}; cat a") == (["a"], False) + assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == ["AGENTS.md"] + + +@pytest.mark.parametrize("command", ["case x in a) cd sub;; esac", "case x in a) :;; b) cd sub;; esac", + "case $x in\n a) echo\n cd sub;;\nesac", "case x in a) source env.sh;; esac"]) +def test_a_directory_builtin_in_a_case_arm_may_move_the_shell(command): + # a case is not followed: the arm may have run + assert shell_reads_moved(f"{command}; cat a") == ([], True) + assert _reads(_events(_call("a", command), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == [] + + +def test_a_case_with_no_directory_builtin_keeps_the_next_read(): + assert shell_reads_moved("case x in a) echo a;; esac; cat a") == (["a"], False) + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From ba846db4397e70145043d821e28c1e7eb1b94313 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:15:00 +0000 Subject: [PATCH 27/42] fix(repo): a brace group holding one read is that read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles the review point that a failing `{ cat AGENTS.md; }`, alone or after `cd sub &&`, kept its read at rc 0, so `guidance_read`, reach and uptake counted a file the agent never saw. `_failed_read` took the group's `{` and `}` as parts of their own: the `}` was the last part, and the read before it was not a `cd`, so the failure could have been another part's. The braces are no longer parts there: a `}` part is skipped and the `{` that opens a group is stripped, so a group holding one read is that read, after any `cd` (`cd sub && { cat AGENTS.md; }`, nested groups too). A group holding another command (`{ make; cat AGENTS.md; }`) keeps its read, as before. The rule is shared, so Codex and Copilot commands get it too; a test each. §4.1, §7 and the changelog fragment say so. Signed-off-by: Anton Volkov --- ...261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 6 +++--- dokimi/dokimi/repos/cli_traces.py | 11 +++++++++-- dokimi/tests/test_repo_cli_traces.py | 9 ++++++++- dokimi/tests/test_repo_events_bash.py | 16 ++++++++++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 335e18d00..c7efd65c7 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index a34a48be5..80bbf7bfd 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -467,7 +467,7 @@ parent's and subagents' together, so a grader need not: | `commands` | `command` events | | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | -| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, a quoted or escaped `"cd"` or `\cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one), its name quoted or escaped too (`"source" "$ENV"`, `\cd "$D"`), in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone, the one command of a brace group (`{ cat AGENTS.md; }`) or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, a quoted or escaped `"cd"` or `\cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one), its name quoted or escaped too (`"source" "$ENV"`, `\cd "$D"`), in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). @@ -1075,8 +1075,8 @@ per scenario). read that was denied or failed is none). For the Claude Code (its Bash tool), Codex and Copilot profiles a shell read counts only as a bare `cat`, `head`, `tail`, `nl` or `sed -n`; a command that is one - such read naming one file, alone or after `cd` commands (`cd sub && cat - AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, `"cd"` too, not + such read naming one file, alone, in a brace group (`{ cat AGENTS.md; }`) + or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, `"cd"` too, not `env cd`), and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads, as the known one does when the other is at a directory not known). A read diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index ab6d92d83..7a9d7d1a5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -568,13 +568,20 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: Otherwise the exit code does not say which read failed (``cat a missing`` prints ``a`` and exits 1), so the reads are kept: also when only one of the files is known (``cd -; cat missing /ws/a``). A part before it that is not surely a ``cd`` (``X="a b" cd sub``, ``time -p cd sub``) keeps them too: its exit may be - the one that failed, and a read is dropped only when that file surely was not printed. + the one that failed, and a read is dropped only when that file surely was not printed. A brace group's braces are + not parts: ``{ cat AGENTS.md; }`` and ``cd sub && { cat AGENTS.md; }`` are one read. """ - if rc in (None, 0) or not reads or not all(_is_cd(part.text) for part in split_script(script)[:-1]): + parts = [p for p in (_ungrouped(part.text) for part in split_script(script)) if p] + if rc in (None, 0) or not reads or not all(_is_cd(p) for p in parts[:-1]): return False return _walk_script(script, "")[1].operands == 1 # the printer's own operands, not the reads kept of them +def _ungrouped(text: str) -> str: + """A part's ``text`` past the ``{`` that open brace groups in it; ``""`` for the ``}`` that closes one.""" + return "" if _GROUP_CLOSE.match(text) else re.sub(r"^(?:\{\s+)+", "", text) + + class _CopilotRun: """What ``copilot_json`` collects across the lines: the first timestamp, tool events by call id, the answer (the final one, else the last message) and the turn count.""" diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index f8ab079c6..4edc7c269 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -535,6 +535,12 @@ def test_codex_failed_command_does_not_count_as_a_read(): assert [e["rc"] for e in t.events if e["kind"] == "command"] == [1, 0, 2] +def test_codex_failed_brace_group_holding_one_read_is_no_read(): + cmd = {"id": "a", "type": "command_execution", "command": "{ cat AGENTS.md; }"} + raw = _jsonl({"type": "item.started", "item": cmd}, {"type": "item.completed", "item": {**cmd, "exit_code": 1}}) + assert [e["kind"] for e in convert("codex-json", raw, None).events] == ["command"] + + def test_codex_failed_command_drops_only_its_own_reads(): """A read that succeeded earlier stays when a later command fails: reach would class the file `fetched`.""" ok = {"type": "command_execution", "command": "cat AGENTS.md"} @@ -644,7 +650,8 @@ def test_copilot_command_exit_code_falls_back_to_success(done, rc): ("cd sub && cat AGENTS.md", {"shellExecution": {"exitCode": 1}}, set()), # the cd or the cat failed # cat names two files, one at a directory not known: the known one may be what it printed ("cd /ws/sub; cd -; cat missing.md /ws/AGENTS.md", {"shellExecution": {"exitCode": 1}}, {"/ws/AGENTS.md"}), -], ids=["exit-1", "success-false", "exit-0", "compound", "cd-then-cat", "two-operands-one-known"]) + ("cd sub && { cat AGENTS.md; }", {"shellExecution": {"exitCode": 1}}, set()), # a group holding one read is it +], ids=["exit-1", "success-false", "exit-0", "compound", "cd-then-cat", "two-operands-one-known", "brace-group"]) def test_copilot_failed_bash_read_is_not_a_read(command, done, reads): raw = _jsonl( {"type": "tool.execution_start", diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index cadf4c1f0..2d526e7c3 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -67,6 +67,22 @@ def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] +@pytest.mark.parametrize("command, path", [("{ cat AGENTS.md; }", "AGENTS.md"), + ("cd sub && { cat AGENTS.md; }", "sub/AGENTS.md"), + ("{ { cat AGENTS.md; }; }", "AGENTS.md"), + ("{ cd sub; cat AGENTS.md; }", "sub/AGENTS.md")]) +def test_a_brace_group_holding_one_read_is_that_read(command, path): + # failed, it read nothing; succeeded, it read its file + events = _events(_call("a", command), _result("a", "cat: AGENTS.md: No such file or directory", True)) + assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] + assert _reads(_events(_call("a", command), _result("a"))) == [path] + + +def test_a_failed_brace_group_with_two_commands_keeps_the_read(): + events = _events(_call("a", "{ make; cat AGENTS.md; }"), _result("a", "error", True)) + assert [(e["kind"], e["rc"]) for e in events] == [("read", 0), ("command", 1)] + + @pytest.mark.parametrize("command", ["cat AGENTS.md && cd sub", "make && cat AGENTS.md", "(cd sub) && cat AGENTS.md", "env cd sub && cat AGENTS.md", # ``env cd`` runs no builtin: some program failed "X=1 time cd sub && cat AGENTS.md", r"\time cd sub && cat AGENTS.md"]) # /usr/bin/time From 87ba8178b313e44a0bc36db97f2ea9cce5cd77c8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:16:28 +0000 Subject: [PATCH 28/42] fix(repo): a refused relative read after a move has no path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles the review question about `cd sub` followed by a refused `cat AGENTS.md`: the denied read was kept with `AGENTS.md`, the path from the start directory, although the shell was then in `sub` or somewhere not known. The counts skip a denied read, but `events.jsonl`, `tool_calls` and the `Read` steps of `compare` / `consistency` carried a wrong path. The read stays, as a call the agent made, but with no `path`: `{"kind": "read", "denied": true}`. An absolute path in the same call keeps its own. Every consumer of read events already reads `path` with `.get` or skips a denied event first (`Transcript.reads()`, `guidance_read`, `skill_usage`, uptake, `run_from_events`, which gives a plain `Read` with no `file_path`); a test pins each. No schema describes the events. §4.1 says a read may have no path. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 +++ dokimi/dokimi/repos/events.py | 21 +++++++++++++----- dokimi/tests/test_repo_events_bash.py | 22 ++++++++++++++++--- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index c7efd65c7..3072de9e8 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 80bbf7bfd..094fb651b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -441,6 +441,9 @@ reads the upstream base (`base_sha`) and the workspace start commit must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's `Agent`/`Task` call that started it. Events without it are the parent's. +- A `read` event may have no `path`: a refused Claude Code Bash read of a + relative path after an earlier call moved the shell, whose file is not + known (`{"kind": "read", "denied": true}`). `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index b62ab0000..09cf68e69 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -24,8 +24,8 @@ next, so once any call ran a ``cd`` in that shell (not in a subshell, a pipeline or a background job), later relative reads are not recorded, the parent's and every subagent's alike: subagents may share the parent's shell, and dropping a read is the safe side either way. A refused call ran nothing: -it moves no shell, and its reads stay, ``denied``, after a move too (a relative path is then taken -from the start directory; ``guidance_read``, reach and skill use skip a denied read). The reads are settled when the trace ends, +it moves no shell, and its reads stay, ``denied``, after a move too (a relative one then has no +``path``: which file it named is not known; ``guidance_read``, reach and skill use skip a denied read). The reads are settled when the trace ends, since a ``permission_denials`` entry comes only in the final result. Token counts cover the same scope as ``cost_usd``: summed over ``modelUsage`` @@ -163,7 +163,8 @@ def _deny(stream: _Stream, tool_id: Any) -> None: def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: """Each Bash call's reads, once every result and denial is in. In the trace's order: - - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``, after a move too; + - a denied call ran nothing: it moves no shell, and its reads stay, ``denied``, after a move too (a relative + one then with no ``path``); - a call that failed and was one read naming one file (``cat AGENTS.md`` of a missing file, after any ``cd``: :func:`~dokimi.repos.cli_traces._failed_read`) drops its reads; - after a call that was not denied and may have moved its shell, only absolute reads are known, in @@ -175,9 +176,10 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: for call in calls: failed = call.result == "error" and _failed_read(call.text, call.named, 1) for read, path in zip(call.reads, call.named, strict=True): - if call.result == "denied": # kept after a move too, its path taken from the start directory - read["denied"] = True - elif failed or (moved and not path.startswith("/")): + unknown = moved and not path.startswith("/") # the shell's directory is not known, nor the file + if call.result == "denied": # kept after a move too, with no path when it was relative + _deny_read(read, unknown) + elif failed or unknown: dropped.add(id(read)) elif call.result: read["rc"] = 0 @@ -186,6 +188,13 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: out.events[:] = [e for e in out.events if id(e) not in dropped] +def _deny_read(read: dict[str, Any], unknown: bool) -> None: + """A refused call's read; ``unknown``: relative, after a move, so the file it named is not known and has no path.""" + read["denied"] = True + if unknown: + del read["path"] + + def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: """The events of one tool_use block: a Bash call's derived reads, then its own event.""" event = _tool_event(block, t, stream.workdir) diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 2d526e7c3..0f8c679b4 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -144,10 +144,26 @@ def test_a_refused_cd_moves_no_shell(refusal, reads): lambda: _denials("b"), ]) def test_a_refused_relative_read_after_a_cd_stays_denied(refusal): - events = _events(_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), refusal()) - assert [(e["kind"], e.get("denied")) for e in events] == [("command", None), ("read", True), ("command", True)] + events = _events(_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md /ws/README.md"), refusal()) + # the relative read's file is not known after the cd: it has no path; the absolute one keeps its own + assert [(e["kind"], e.get("path"), e.get("denied")) for e in events] == [ + ("command", None, None), ("read", None, True), ("read", "README.md", True), ("command", None, True)] + assert "path" not in events[1] got = process_metrics(events, lambda p: p.endswith("AGENTS.md")) - assert got["tool_calls"] == 3 and got["guidance_read"] == [] + assert got["tool_calls"] == 4 and got["guidance_read"] == [] + + +def test_a_read_with_no_path_is_no_read_to_any_consumer(): + from dokimi.context.consistency import run_from_events + from dokimi.repos.arms import skill_usage + from dokimi.repos.transcript import Transcript + from dokimi.repos.uptake import arm_uptake + events = _events(_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), _denials("b")) + assert events[1] == {"t": 5.0, "kind": "read", "denied": True} + assert Transcript(events=events).reads() == set() + assert skill_usage(events, ["lint"], None)["skills_used"] == [] + assert arm_uptake(events, ["AGENTS.md"], [], WS)["reads"] == 0 + assert [(e.action, e.input) for e in run_from_events(events).events if e.tool == "Read"] == [("Read", {})] @pytest.mark.parametrize("command", ["cat AGENTS.md | head", "grep -n rule AGENTS.md", "cat AGENTS.md > copy.md"]) From 226a04cede787bb8013fac990f5e1314deb9a1e9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:21:20 +0000 Subject: [PATCH 29/42] feat(repo): count the shell reads at a directory not known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles the review point that one `source .venv/bin/activate` drops every later relative Bash read of the run with nothing in `result.json` to say so: `guidance_read: []` then reads as "no guidance read", not "not known". `result.json` `process` gains `unknown_reads`, an int, `0` when there are none: the relative reads a trace made at a directory the converter could not follow, which give no read event. The walker counts the relative files a printing command names once its directory is lost in the command (`cd -; cat a`, `source env.sh && cat a`), and `shell_walk` returns that count with the reads and the move. The Codex and Copilot converters add it per command; the Claude Code converter adds it per Bash call that was not refused, plus each relative read it drops after an earlier call moved the shell (a failed one-file read is not counted: it failed). `Transcript.unknown_reads` carries it to `process_metrics`. No schema describes `process`; §4.1 documents the key next to `guidance_read`, and the changelog fragment gains an "Added" entry. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 4 ++ dokimi/docs/repo-evaluation-contract.md | 1 + dokimi/dokimi/repos/cli_traces.py | 44 ++++++++++++++----- dokimi/dokimi/repos/events.py | 13 ++++-- dokimi/dokimi/repos/process_metrics.py | 8 +++- dokimi/dokimi/repos/runner.py | 11 ++--- dokimi/dokimi/repos/transcript.py | 1 + dokimi/tests/test_repo_cli_traces.py | 11 +++++ dokimi/tests/test_repo_events_bash.py | 19 ++++++++ dokimi/tests/test_repo_process_metrics.py | 3 +- dokimi/tests/test_repo_run.py | 12 ++++- 11 files changed, 102 insertions(+), 25 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 3072de9e8..a38f6d253 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -6,3 +6,7 @@ SPDX-License-Identifier: Apache-2.0 ### Changed - **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. + +### Added + +- **`process.unknown_reads` in `result.json`** — the relative shell reads a trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call), which give no `read` event, for Claude Code, Codex and Copilot alike; `0` when there are none. A `guidance_read` of `[]` with `unknown_reads` above 0 means not known, not none read. `docs/repo-evaluation-contract.md` §4.1 says so. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 094fb651b..bbfeb87a6 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -471,6 +471,7 @@ parent's and subagents' together, so a grader need not: | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | | `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone, the one command of a brace group (`{ cat AGENTS.md; }`) or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, a quoted or escaped `"cd"` or `\cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one), its name quoted or escaped too (`"source" "$ENV"`, `\cd "$D"`), in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | +| `unknown_reads` | the relative shell reads, of calls not refused, that the trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call): they have no `read` event, so `guidance_read` may miss them; `0` when there are none | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 7a9d7d1a5..a0c7292e9 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -106,8 +106,13 @@ def shell_reads_moved(script: str, cwd: str = "") -> tuple[list[str], bool]: ``pushd``, ``popd``, ``eval``, ``source`` or ``.`` in that shell, not in a subshell (``( ... )``, a pipeline or a backgrounded command, list or group: ``cd sub &``, ``{ cd sub; } | cat``, ``cd sub && make &``, ``cd "$X" &``, ``cd sub | cat``, ``{ cd $(pwd); } &``).""" + return shell_walk(script, cwd)[:2] + + +def shell_walk(script: str, cwd: str = "") -> tuple[list[str], bool, int]: + """:func:`shell_reads_moved`, and how many relative files its printing commands named at a directory not known.""" reads, walk = _walk_script(script, cwd) - return reads, walk.shell_moved + return reads, walk.shell_moved, walk.unknown def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: @@ -159,6 +164,7 @@ def __init__(self, here: str | None) -> None: self.function = 0 # the brace depth inside a function definition self.case = 0 # the depth inside ``case ... esac`` self.operands: int | None = None # how many files the last printing command named + self.unknown = 0 # the relative files printing commands named at a directory not known def spot(self) -> _Spot: return self.here, self.shell_moved, self.moved @@ -268,14 +274,20 @@ def command(self, words: list[str], after: str, chained: bool = False, timed: bo if after != "&": # a backgrounded cd runs in a subshell: this shell stays where it was self.here, self.shell_moved, self.moved = _cd(self.here, words[1:]), True, True elif name in _PRINTERS: - if _asks_for_help(words[1:]): # ``cat --help AGENTS.md`` prints the help and opens no file - return [] - files = _sed_files(words[1:]) if name == "sed" else _print_files(name, words[1:]) - self.operands = len(files) # all it names, those at an unknown directory too - # ``env -C dir cat f`` reads dir/f, which is not known: only an absolute path is the same - return [self.resolve(f) for f in files if f.startswith("/") or (self.here is not None and not chdir)] + return self.printed(name, words[1:], chdir) return [] + def printed(self, name: str, args: list[str], chdir: bool) -> list[str]: + """The files a printing command reads; ``unknown`` counts those it names at a directory not known.""" + if _asks_for_help(args): # ``cat --help AGENTS.md`` prints the help and opens no file + return [] + files = _sed_files(args) if name == "sed" else _print_files(name, args) + self.operands = len(files) # all it names, those at an unknown directory too + # ``env -C dir cat f`` reads dir/f, which is not known: only an absolute path is the same + known = [f for f in files if f.startswith("/") or (self.here is not None and not chdir)] + self.unknown += len(files) - len(known) + return [self.resolve(f) for f in known] + def resolve(self, f: str) -> str: return posixpath.normpath(f if f.startswith("/") else posixpath.join(self.here or "", f)) @@ -506,7 +518,7 @@ def _codex_command_started(out: Transcript, item: dict[str, Any], workdir: str | event_reads: dict[int, list[dict[str, Any]]]) -> None: script = unwrap_shell(str(item.get("command") or "")) read_events = [{"t": None, "kind": "read", "path": relative_path(path, workdir)} - for path in shell_reads(script, cwd)] + for path in _counted_reads(out, script, cwd)] out.events.extend(read_events) event: dict[str, Any] = {"t": None, "kind": "command", "text": script, "rc": None} event_reads[id(event)] = read_events @@ -551,7 +563,7 @@ def _codex_command_finished(out: Transcript, item: dict[str, Any], event: dict[s def _codex_command_unstarted(out: Transcript, item: dict[str, Any], script: str, workdir: str | None, cwd: str) -> None: """Fallback for item.completed without item.started.""" - paths = shell_reads(script, cwd) + paths = _counted_reads(out, script, cwd) if not _failed_read(script, paths, item.get("exit_code")): for path in paths: out.events.append({"t": None, "kind": "read", "path": relative_path(path, workdir)}) @@ -561,6 +573,13 @@ def _codex_command_unstarted(out: Transcript, item: dict[str, Any], script: str, out.events.append(event) +def _counted_reads(out: Transcript, script: str, cwd: str) -> list[str]: + """:func:`shell_reads`; ``out.unknown_reads`` counts the files it named at a directory not known.""" + reads, _, unknown = shell_walk(script, cwd) + out.unknown_reads += unknown + return reads + + def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: """A non-zero exit of a command that is one part naming one file, alone or after ``cd`` parts (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read. @@ -616,7 +635,7 @@ def apply(self, out: Transcript, rec: dict[str, Any], t: float | None, workdir: elif kind == "assistant.message": self.message(data) elif kind == "tool.execution_start": - events = _copilot_tool(data, t, workdir, cwd) + events = _copilot_tool(data, t, workdir, cwd, out) out.events.extend(events) self.by_id[str(data.get("toolCallId"))] = events elif kind == "tool.execution_complete": @@ -650,13 +669,14 @@ def copilot_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: return out -def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cwd: str = "") -> list[dict[str, Any]]: +def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cwd: str, + out: Transcript) -> list[dict[str, Any]]: name = str(data.get("toolName") or "") args = data.get("arguments") if name in _COPILOT_SHELL: command = str((args or {}).get("command", "")) if isinstance(args, dict) else "" # only bash is a POSIX script; a PowerShell command is not parsed for the files it reads - scripts = shell_reads(command, cwd) if name == "bash" else [] + scripts = _counted_reads(out, command, cwd) if name == "bash" else [] reads = [{"t": t, "kind": "read", "path": relative_path(p, workdir)} for p in scripts] return [*reads, {"t": t, "kind": "command", "text": command, "rc": None}] if name in _COPILOT_READ | _COPILOT_EDIT and isinstance(args, dict): diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 09cf68e69..129302839 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -44,7 +44,7 @@ from dokimi.context.trace import parse_trace -from .cli_traces import _failed_read, codex_json, copilot_json, shell_reads_moved +from .cli_traces import _failed_read, codex_json, copilot_json, shell_walk from .transcript import Transcript, _parse_time, clip_output, relative_path TRACE_FORMATS = ("claude-stream-json", "codex-json", "copilot-json", "text") @@ -79,6 +79,7 @@ class _BashCall: named: list[str] # the files its command prints, from the start directory reads: list[dict[str, Any]] # their read events, in place before the command event moves: bool # the command may leave its shell in another directory + unknown: int = 0 # the relative files it named after a directory change it could not follow, in itself result: str | None = None # "ok", "error" or "denied"; None while nothing came back @@ -170,17 +171,23 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: - after a call that was not denied and may have moved its shell, only absolute reads are known, in every agent: a subagent may share the parent's shell; - else the reads get ``rc`` 0, since a failure does not say which file failed (``cat a missing``). + + ``out.unknown_reads`` counts the relative reads of the calls not denied whose directory was not known: dropped + after a move, or after a directory change in the call that the walker could not follow (``cd -; cat a``). """ moved = False # the shell may have left its start directory dropped: set[int] = set() for call in calls: failed = call.result == "error" and _failed_read(call.text, call.named, 1) + if call.result != "denied": + out.unknown_reads += call.unknown for read, path in zip(call.reads, call.named, strict=True): unknown = moved and not path.startswith("/") # the shell's directory is not known, nor the file if call.result == "denied": # kept after a move too, with no path when it was relative _deny_read(read, unknown) elif failed or unknown: dropped.add(id(read)) + out.unknown_reads += not failed # dropped as its directory was not known, not as it failed elif call.result: read["rc"] = 0 moved |= call.moves and call.result != "denied" @@ -211,9 +218,9 @@ def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: st def _bash_call(event: dict[str, Any], stream: _Stream) -> _BashCall: """A Bash command's call, with a read event for each file it names; :func:`_settle_bash` keeps or drops them.""" - named, moves = shell_reads_moved(str(event["text"]), stream.cwd) + named, moves, unknown = shell_walk(str(event["text"]), stream.cwd) reads = [{"t": event["t"], "kind": "read", "path": relative_path(p, stream.workdir)} for p in named] - stream.bash[id(event)] = call = _BashCall(str(event["text"]), named, reads, moves) + stream.bash[id(event)] = call = _BashCall(str(event["text"]), named, reads, moves, unknown) return call diff --git a/dokimi/dokimi/repos/process_metrics.py b/dokimi/dokimi/repos/process_metrics.py index fd6336780..ba09f24d6 100644 --- a/dokimi/dokimi/repos/process_metrics.py +++ b/dokimi/dokimi/repos/process_metrics.py @@ -14,7 +14,10 @@ - ``guidance_read``: the sorted workspace paths of ``read`` events (a read tool, or a shell command that only prints files, ``cat`` / ``head`` / ``tail`` / ``nl`` / ``sed -n``; a pipe, ``grep`` or ``rg`` is not one) not denied that the manifest's guidance globs match; a path outside the workspace (absolute, or with ``..``) is - no repository file. + no repository file; +- ``unknown_reads``: the relative shell reads the trace made at a directory the converter could not follow + (after ``source env.sh``, say), which have no ``read`` event: ``guidance_read`` may miss them. It comes from + the converter (:attr:`~dokimi.repos.transcript.Transcript.unknown_reads`), not from the events. ``counters`` holds the manifest's ``process_metrics``: for each name, how many ``command`` events its regex finds a match in (``re.search``). A counter matches the command text unless it names @@ -77,7 +80,7 @@ def _workspace_path(path: str) -> str | None: def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[str], bool], - counters: Mapping[str, Counter] | None = None) -> dict[str, Any]: + counters: Mapping[str, Counter] | None = None, unknown_reads: int = 0) -> dict[str, Any]: """The ``process`` object of ``result.json`` for one run's events; ``is_guidance`` tells a guidance path.""" events = list(events) commands = [e for e in events if e.get("kind") == "command"] @@ -90,5 +93,6 @@ def process_metrics(events: Iterable[Mapping[str, Any]], is_guidance: Callable[[ "failed_commands": sum(e.get("rc") not in (0, None) and not e.get("denied") for e in commands), "denied_commands": sum(bool(e.get("denied")) for e in commands), "guidance_read": sorted(p for p in reads if is_guidance(p)), + "unknown_reads": unknown_reads, "counters": {name: sum(c.hits(e) for e in commands) for name, c in (counters or {}).items()}, } diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a0a122128..96ee37dca 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -273,7 +273,7 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, subagent_calls=transcript.subagent_calls, plugins=transcript.plugins, over_budget=cost is not None and cost > caps["budget_usd"], trace_errors=transcript.errors[:5]) if events is not None: # after the cost is recorded: a metric that fails must not cost the cell its record - process = _process_or_null(ctx, events, agent["trace_errors"]) + process = _process_or_null(ctx, events, agent["trace_errors"], transcript.unknown_reads) if meta.get("skills") and started: skills = skill_usage(transcript.events, [s["name"] for s in meta["skills"]], task.expected_skill) graded = run_entry(ctx, "grade", task, rd, grade_network, container_name(rd.name, "grade")) @@ -298,17 +298,18 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, return record -def run_process_metrics(ctx: EvalContext, events: list[dict[str, Any]]) -> dict[str, Any]: +def run_process_metrics(ctx: EvalContext, events: list[dict[str, Any]], unknown_reads: int = 0) -> dict[str, Any]: """``result.json`` ``process``: the framework's counts and the manifest's ``process_metrics`` (contract §4.1).""" globs = ctx.manifest.guidance_globs return process_metrics(events, lambda path: any(match_glob(g, path) for g in globs), - ctx.manifest.process_metrics) + ctx.manifest.process_metrics, unknown_reads) -def _process_or_null(ctx: EvalContext, events: list[dict[str, Any]], errors: list[str]) -> dict[str, Any] | None: +def _process_or_null(ctx: EvalContext, events: list[dict[str, Any]], errors: list[str], + unknown_reads: int = 0) -> dict[str, Any] | None: """The cell's ``process``, or None when its metrics raise: a best-effort count must not fail the task.""" try: - return run_process_metrics(ctx, events) + return run_process_metrics(ctx, events, unknown_reads) except Exception as exc: # noqa: BLE001 # a metric that fails is recorded, the grade still counts errors.append(f"process metrics failed ({type(exc).__name__}): {str(exc)[:200]}") return None diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 8b9efacf4..445113a69 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -26,6 +26,7 @@ class Transcript: answer: str = "" agent_messages: list[str] = field(default_factory=list) # codex: every agent message in order; answer is the last events: list[dict[str, Any]] = field(default_factory=list) + unknown_reads: int = 0 # relative shell reads at a directory the converter could not follow: no read event cost_usd: float | None = None num_turns: int | None = None result_seen: bool = False # the harness wrote its final result record, so the agent started diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index 4edc7c269..ca5ee5169 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -535,6 +535,17 @@ def test_codex_failed_command_does_not_count_as_a_read(): assert [e["rc"] for e in t.events if e["kind"] == "command"] == [1, 0, 2] +def test_codex_and_copilot_count_a_read_at_a_directory_not_known(): + codex = {"id": "a", "type": "command_execution", "command": "cd -; cat AGENTS.md"} + raw = _jsonl({"type": "item.started", "item": codex}, {"type": "item.completed", "item": {**codex, "exit_code": 0}}, + {"type": "item.completed", "item": {**codex, "id": "b", "command": "cd -; cat a.md", "exit_code": 0}}) + assert convert("codex-json", raw, None).unknown_reads == 2 + copilot = {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "cd -; cat AGENTS.md"}} + assert convert("copilot-json", _jsonl({"type": "tool.execution_start", "data": copilot}), None).unknown_reads == 1 + assert convert("copilot-json", _jsonl({"type": "tool.execution_start", "data": { + **copilot, "arguments": {"command": "cat AGENTS.md"}}}), None).unknown_reads == 0 + + def test_codex_failed_brace_group_holding_one_read_is_no_read(): cmd = {"id": "a", "type": "command_execution", "command": "{ cat AGENTS.md; }"} raw = _jsonl({"type": "item.started", "item": cmd}, {"type": "item.completed", "item": {**cmd, "exit_code": 1}}) diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 0f8c679b4..7562068fe 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -332,6 +332,25 @@ def test_a_bash_call_with_no_result_keeps_its_read_without_an_rc(): assert events[0] == {"t": 5.0, "kind": "read", "path": "AGENTS.md"} +def _process(*lines: dict) -> dict: + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + t = convert("claude-stream-json", "\n".join(json.dumps(x) for x in (first, *lines)), WS) + return process_metrics(t.events, lambda p: p.endswith("AGENTS.md"), unknown_reads=t.unknown_reads) + + +@pytest.mark.parametrize("lines, unknown, guidance", [ + ((_call("a", "source .venv/bin/activate && pytest"), _result("a"), _call("b", "cat AGENTS.md"), _result("b")), 1, []), + ((_call("a", "cd - && cat AGENTS.md README.md"), _result("a")), 2, []), # within the call: cd - is not followed + ((_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md /ws/a.md"), _result("b", "x", True)), 1, []), + ((_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b", "x", True)), 0, []), # it failed + ((_call("a", "cd sub"), _result("a"), _call("b", "cd -; cat AGENTS.md"), _denials("b")), 0, []), # nothing ran + ((_call("a", "cat AGENTS.md"), _result("a"), _call("b", "cd sub && make"), _result("b")), 0, ["AGENTS.md"]), +]) +def test_a_read_at_a_directory_not_known_is_counted(lines, unknown, guidance): + got = _process(*lines) + assert (got["unknown_reads"], got["guidance_read"]) == (unknown, guidance) + + def test_guidance_printed_through_bash_is_guidance_read(): events = _events(_call("b", "cat AGENTS.md"), _result("b", "cat: AGENTS.md: No such file or directory", True), _call("c", "cat README.md"), _result("c"), diff --git a/dokimi/tests/test_repo_process_metrics.py b/dokimi/tests/test_repo_process_metrics.py index 4162025ba..4f07f311b 100644 --- a/dokimi/tests/test_repo_process_metrics.py +++ b/dokimi/tests/test_repo_process_metrics.py @@ -52,7 +52,8 @@ def test_built_in_counts(): def test_no_events_counts_zero(): assert process_metrics([], _guidance) == {"tool_calls": 0, "commands": 0, "failed_commands": 0, - "denied_commands": 0, "guidance_read": [], "counters": {}} + "denied_commands": 0, "guidance_read": [], "unknown_reads": 0, + "counters": {}} def test_counters_search_text_output_or_both(): diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 827496e88..1bd5db889 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -65,7 +65,7 @@ def test_run_isolates_the_agent_and_grades_each_cell(setup, capsys): assert result["agent"]["cost_usd"] == 0.01 and result["agent"]["num_turns"] == 3 assert result["model_resolved"] == "m-1" assert result["process"] == {"tool_calls": 1, "commands": 0, "failed_commands": 0, "denied_commands": 0, - "guidance_read": [], + "guidance_read": [], "unknown_reads": 0, "counters": {"bazel": 0, "icpx_missing": 0}} agents = [s for s in setup["runner"].specs if s.argv and s.argv[0] == f"{HARNESS_DIR}/fake-agent" @@ -100,10 +100,18 @@ def test_process_metrics_reach_result_json(setup, monkeypatch): _run(setup, "--arm", "guidance:raw", "--task", "fix_add") rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) assert rec["process"] == {"tool_calls": 3, "commands": 1, "failed_commands": 1, "denied_commands": 0, - "guidance_read": ["AGENTS.md"], + "guidance_read": ["AGENTS.md"], "unknown_reads": 0, "counters": {"bazel": 1, "icpx_missing": 1}} +def test_a_read_at_a_directory_not_known_is_counted_in_result_json(setup, monkeypatch): + calls = [["Bash", {"command": "source .venv/bin/activate && pytest"}], ["Bash", {"command": "cat AGENTS.md"}]] + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps(calls)) + _run(setup, "--arm", "guidance:raw", "--task", "fix_add") + rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) + assert rec["process"]["guidance_read"] == [] and rec["process"]["unknown_reads"] == 1 + + def test_an_unreadable_trace_leaves_process_null_and_still_grades(setup, monkeypatch): def refuse(*args, **kwargs): raise ValueError("unreadable trace") From c51f68a499b1c560a6a83e0d4081abb039069b9a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:21:40 +0000 Subject: [PATCH 30/42] test(repo): pin a failed cd continued over a line as no read Answers the review point that `_failed_read` splits the raw script and so would keep a failed `cd \sub && cat AGENTS.md`. It does not: `split_script` keeps a line continuation inside its part, so the first part is still the `cd`, and the failed read gives no read. No code change; the failed cd-then-cat test gains that form and `cd sub \&& cat AGENTS.md`, which no test covered. Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_events_bash.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 7562068fe..4a701ba57 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -61,7 +61,8 @@ def test_a_failed_cat_of_one_file_is_no_read(): "time cd sub && cat AGENTS.md", "command cd sub && cat AGENTS.md", "builtin cd sub && cat AGENTS.md", "X=1 cd sub && cat AGENTS.md", '"cd" sub && cat AGENTS.md', r"\cd sub && cat AGENTS.md", - "builtin 'cd' sub && cat AGENTS.md"]) # a quoted or escaped name is the builtin + "builtin 'cd' sub && cat AGENTS.md", # a quoted or escaped name is the builtin + "cd \\\nsub && cat AGENTS.md", "cd sub \\\n&& cat AGENTS.md"]) # continued lines def test_a_failed_cd_then_cat_of_one_file_is_no_read(command): events = _events(_call("a", command), _result("a", "cat: AGENTS.md: No such file or directory", True)) assert [(e["kind"], e["rc"]) for e in events] == [("command", 1)] From b52a3e10da2c3eff25fd41203b152110cfacac1e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 14:24:43 +0000 Subject: [PATCH 31/42] fix(repo): a hidden group's reads are not counted as unknown Settles the review point that `process.unknown_reads` counted prints whose output never reached the agent. In `{ source env.sh; cat AGENTS.md; } > /dev/null`, or a group feeding a pipe, the walker drops the group's read events, as its output is not shown, but kept the count `printed()` had added for the group's relative files at a directory not known: `shell_walk` gave no read and `unknown=1`, which the three converters carry into `result.json`, so an empty `guidance_read` looked uncertain. An open brace group now saves the unknown count with its read mark, and the close that drops the group's reads rolls the count back to it. A nested hidden group rolls back only its own count, and a group whose output is shown keeps its count. Tests through `shell_walk` and through the Claude Code converter to `process_metrics`. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/cli_traces.py | 16 +++++++++------- dokimi/tests/test_repo_cli_traces.py | 15 ++++++++++++++- dokimi/tests/test_repo_events_bash.py | 3 +++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index a0c7292e9..6dfe67045 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -156,9 +156,9 @@ def __init__(self, here: str | None) -> None: self.moved = False # a cd was seen in this shell: after ||, the directory is not known self.shell_moved = False # a directory builtin ran in the script's shell, not in a subshell self.gated = False # a cd after && or || was seen in this chain: it may not have run for the next chain - # the open brace groups: the spot before it, in a pipeline, index in the reads where it began, the - # enclosing && / || list's (list_start, listed), which the group's own separators do not end - self.groups: list[tuple[_Spot, bool, int, tuple[_Spot, bool]]] = [] + # the open brace groups: the spot before it, in a pipeline, (index in the reads, ``unknown``) where it began, + # the enclosing && / || list's (list_start, listed), which the group's own separators do not end + self.groups: list[tuple[_Spot, bool, tuple[int, int], tuple[_Spot, bool]]] = [] self.list_start: _Spot = (here, False, False) # the spot where the && / || list began self.listed = False # that list has more than one command self.function = 0 # the brace depth inside a function definition @@ -214,8 +214,9 @@ def lose(self, own_shell: bool = True) -> None: def track_groups(self, bare: str, after: str, out: list[str]) -> None: """A brace group opening or closing in ``bare``; one that ran in a subshell restores the directory. A group is one command of the list around it: ``cd sub && { a; b; } &`` backgrounds the whole list. The reads of a group - whose output feeds a pipe after it, or a file, are dropped from ``out``: its output is not the agent's. Each - brace of ``bare`` counts, in order (``{ { cd x``, ``} | { cat a``).""" + whose output feeds a pipe after it, or a file, are dropped from ``out``, and so are those it counted in + ``unknown``: its output is not the agent's. Each brace of ``bare`` counts, in order (``{ { cd x``, + ``} | { cat a``).""" braces = [(m.end() - 1, m) for m in _GROUP_OPEN.finditer(bare)] + [(m.end(), m) for m in _GROUP_CLOSE.finditer(bare)] for _, m in sorted(braces, key=lambda b: b[0]): if m.re is _GROUP_OPEN: @@ -225,14 +226,15 @@ def track_groups(self, bare: str, after: str, out: list[str]) -> None: def open_group(self, before_brace: str, mark: int) -> None: piped = bool(_PIPE.search(before_brace)) - self.groups.append((self.spot(), piped, mark, (self.list_start, self.listed))) + self.groups.append((self.spot(), piped, (mark, self.unknown), (self.list_start, self.listed))) if piped: # the group runs in a pipeline's subshell: where it ends up is not tracked self.here = None def close_group(self, after_close: str, redirects: str, after: str, out: list[str]) -> None: - before, piped, mark, (self.list_start, self.listed) = self.groups.pop() + before, piped, (mark, unknown), (self.list_start, self.listed) = self.groups.pop() if _PIPE.search(after_close) or _STDOUT_FILE.search(redirects) or _STDOUT_AFTER_STDERR.search(redirects): del out[mark:] + self.unknown = unknown if piped or _PIPE.search(after_close) or after == "&": # the group ran in a pipeline's or a job's subshell self.back(before) diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index ca5ee5169..dc842f7db 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -11,7 +11,7 @@ import pytest -from dokimi.repos.cli_traces import _LONG_VALUE, shell_reads, unwrap_shell +from dokimi.repos.cli_traces import _LONG_VALUE, shell_reads, shell_walk, unwrap_shell from dokimi.repos.events import convert # --- traces ------------------------------------------------------------------ @@ -546,6 +546,19 @@ def test_codex_and_copilot_count_a_read_at_a_directory_not_known(): **copilot, "arguments": {"command": "cat AGENTS.md"}}}), None).unknown_reads == 0 +@pytest.mark.parametrize("command, unknown", [ + ("{ source env.sh; cat AGENTS.md; } > /dev/null", 0), # the group's output is hidden: not the agent's + ("{ source env.sh; cat AGENTS.md; } | head", 0), + ("source env.sh; { cat AGENTS.md; } > /dev/null", 0), + ("{ source env.sh; { cat a; } > /dev/null; cat b; }", 1), # only the inner group is hidden + ("{ source env.sh; cat AGENTS.md; }", 1), + ("source env.sh; { cat AGENTS.md; }", 1), +]) +def test_a_hidden_groups_reads_are_not_counted_at_a_directory_not_known(command, unknown): + reads, _, got = shell_walk(command) + assert (reads, got) == ([], unknown) + + def test_codex_failed_brace_group_holding_one_read_is_no_read(): cmd = {"id": "a", "type": "command_execution", "command": "{ cat AGENTS.md; }"} raw = _jsonl({"type": "item.started", "item": cmd}, {"type": "item.completed", "item": {**cmd, "exit_code": 1}}) diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 4a701ba57..72ad2640f 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -346,6 +346,9 @@ def _process(*lines: dict) -> dict: ((_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b", "x", True)), 0, []), # it failed ((_call("a", "cd sub"), _result("a"), _call("b", "cd -; cat AGENTS.md"), _denials("b")), 0, []), # nothing ran ((_call("a", "cat AGENTS.md"), _result("a"), _call("b", "cd sub && make"), _result("b")), 0, ["AGENTS.md"]), + ((_call("a", "{ source env.sh; cat AGENTS.md; } > /dev/null"), _result("a")), 0, []), # its output is hidden + ((_call("a", "{ source env.sh; cat AGENTS.md; } | head"), _result("a")), 0, []), + ((_call("a", "{ source env.sh; cat AGENTS.md; }"), _result("a")), 1, []), ]) def test_a_read_at_a_directory_not_known_is_counted(lines, unknown, guidance): got = _process(*lines) From 8b086655fedd15c4d7083101a07e6ed227c4b843 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 14:32:13 +0000 Subject: [PATCH 32/42] docs(repo): reviewing.md says which shell reads guidance_read counts The "Delivery first" step that came in with #365 said that process:guidance_read counts Read-tool reads only. On this branch a Claude Code Bash call, Codex command or Copilot bash command that only prints files counts too, and process:unknown_reads counts the relative reads dropped after the shell moved. Say that, and keep the pointer to events.jsonl, agent.stdout or a process_metrics counter for the rest. Signed-off-by: Anton Volkov --- dokimi/docs/reviewing.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 43c6f136b..c8bad60b2 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -197,8 +197,12 @@ these, in order, before writing what it showed (the fields: that tree, not the cells'. Where a credit turns on it, read the cells' traces too. A nested `AGENTS.md` loads later if at all (reach's `cwd=`, `read=` show what loads it). `process:guidance_read` - counts Read-tool reads only: 0 from an agent that used Bash is not "never - read"; look for a `cat` or `grep` of a guidance file in `events.jsonl` or + counts a read tool's reads and a Claude Code Bash call, Codex command or + Copilot `bash` command that only prints files (§7's list: a bare `cat`, + `head`, `tail`, `nl` or `sed -n`), not a pipe, `grep` or `rg`, nor a + relative read after the shell moved; `process:unknown_reads` counts those + dropped relative reads, so `guidance_read: []` with `unknown_reads` above 0 + is not "never read". For the rest, look in `events.jsonl` or `agent.stdout`, or add a `process_metrics` counter. Name what was delivered as what was tested ("the root `AGENTS.md`", not "the guidance"). - **A leakage cut.** A task is out of sample for a guidance revision only when From 77a3bdd84e37541f04f4adfa2b8ba655fb2fb008 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 19:13:33 +0000 Subject: [PATCH 33/42] fix(repo): repo report and compare show process:unknown_reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review point 4458839235 on #369: result.json's process block has carried unknown_reads since 226a04c, but process_counts.PROCESS_KEYS did not list it, so process_values({"process": {"guidance_read": [], "unknown_reads": 3}}) gave only process:guidance_read, and a batch whose every cell sourced a venv before cat AGENTS.md reported a guidance_read mean of 0 and nothing else. reviewing.md already named process:unknown_reads, which neither command printed. PROCESS_KEYS now lists unknown_reads right after guidance_read, so repo report's process line and repo compare's descriptive measures carry it in the contract's §4.1 order. The tests pin process_values on the reviewer's input, the report line and the compare row order. Signed-off-by: Anton Volkov --- ...1102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/dokimi/repos/process_counts.py | 2 +- dokimi/tests/test_repo_process_counts.py | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index a38f6d253..060c1f6a9 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -9,4 +9,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **`process.unknown_reads` in `result.json`** — the relative shell reads a trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call), which give no `read` event, for Claude Code, Codex and Copilot alike; `0` when there are none. A `guidance_read` of `[]` with `unknown_reads` above 0 means not known, not none read. `docs/repo-evaluation-contract.md` §4.1 says so. +- **`process.unknown_reads` in `result.json`** — the relative shell reads a trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call), which give no `read` event, for Claude Code, Codex and Copilot alike; `0` when there are none. A `guidance_read` of `[]` with `unknown_reads` above 0 means not known, not none read. `repo report` and `repo compare` show it as `process:unknown_reads`, after `process:guidance_read`. `docs/repo-evaluation-contract.md` §4.1 says so. diff --git a/dokimi/dokimi/repos/process_counts.py b/dokimi/dokimi/repos/process_counts.py index 8ec2a173d..8a0655468 100644 --- a/dokimi/dokimi/repos/process_counts.py +++ b/dokimi/dokimi/repos/process_counts.py @@ -22,7 +22,7 @@ #: ``result.json`` ``process`` keys that are counts (``guidance_read`` is a list of paths, counted). -PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read") +PROCESS_KEYS = ("tool_calls", "commands", "failed_commands", "denied_commands", "guidance_read", "unknown_reads") def process_lists(recs: list[dict[str, Any]]) -> dict[str, list[float | None]]: diff --git a/dokimi/tests/test_repo_process_counts.py b/dokimi/tests/test_repo_process_counts.py index 45174c659..50921ca75 100644 --- a/dokimi/tests/test_repo_process_counts.py +++ b/dokimi/tests/test_repo_process_counts.py @@ -35,14 +35,16 @@ def process(name): raw = "__guidance-raw__" in name return {"tool_calls": 10 if raw else 14, "commands": 4, "failed_commands": 0 if raw else 2, "denied_commands": 0, "guidance_read": ["AGENTS.md"] if raw else [], - "counters": {"bazel": 3 if raw else 1}} + "unknown_reads": 0 if raw else 2, "counters": {"bazel": 3 if raw else 1}} _with_process(tmp_path, process) summary = compare([tmp_path], "guidance:none") measures = [r["measure"] for r in summary["comparisons"]] assert measures[:3] == ["gate", "cost_usd", "turns"] assert measures[3:] == ["process:tool_calls", "process:commands", "process:failed_commands", - "process:denied_commands", "process:guidance_read", "counter:bazel"] + "process:denied_commands", "process:guidance_read", "process:unknown_reads", + "counter:bazel"] + assert _row(summary, "process:unknown_reads")["delta"]["mean"] == pytest.approx(-2.0) row = _row(summary, "process:tool_calls") assert (row["n_tasks"], row["baseline_mean"], row["treatment_mean"]) == (3, 14.0, 10.0) assert row["delta"]["mean"] == pytest.approx(-4.0) and row["down"] == 3 and row["p_holm"] is None @@ -146,6 +148,18 @@ def test_report_shows_mean_process_counters(tmp_path): assert text.count("process ") == 1 +def test_unknown_reads_is_a_process_measure_beside_guidance_read(tmp_path): + # An empty guidance_read with reads the converter could not place is not "never read": both are shown. + assert process_values({"process": {"guidance_read": [], "unknown_reads": 3}}) == { + "process:guidance_read": 0.0, "process:unknown_reads": 3.0} + for rep in (1, 2): + _cell(tmp_path, "a", "guidance:raw", rep) + _with_process(tmp_path, lambda name: {"tool_calls": 4, "guidance_read": [], "unknown_reads": int(name[-1]), + "counters": {"bazel": 1}}) + assert "process tool_calls 4, guidance_read 0, unknown_reads 1.5, counter:bazel 1" in \ + format_report(summarize(tmp_path)) + + def test_process_values_skip_what_is_not_a_count(): assert process_values({}) == {} and process_values({"process": None}) == {} got = process_values({"process": {"tool_calls": True, "commands": float("nan"), "failed_commands": "3", From b6ce65f1f06c7eeb11a363d5c3130f473af47779 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 19:18:09 +0000 Subject: [PATCH 34/42] fix(repo): time with assignments, nested case, braces on a case part Review point 4458839251 on #369: three forms of the round-9 case and time handling kept a read at a wrong path or a hidden one. - time X=1 cd sub; cat AGENTS.md gave (['AGENTS.md'], False, 0), but bash's time keyword times the whole simple command, so the cd runs in this shell. A time prefix may now be followed by NAME=value words before the directory builtin: shell_words._prefix_runs_dir where _command_words skips the prefix, and _command_at past time's options (so time -p X=1 cd sub counts as a move). It now gives (['sub/AGENTS.md'], True, 0); command X=1 cd and builtin X=1 cd, which run no cd, are unchanged. - case $x in a) case $y in b) :;; esac; cd sub;; esac; cat AGENTS.md read sub/AGENTS.md: the inner esac ended the outer case. Inside a case each case word now opens one more level and only a leading esac closes one, so a miscount stays inside (reads dropped). It now gives ([], True, 1). - { cat AGENTS.md; { case $x in a) :;; esac; }; } > /dev/null kept AGENTS.md: in_case returned before track_groups, so the brace opened on the case part was not tracked and the outer group's redirect closed nothing. A case part now tracks its braces first (before the lose() its builtin causes), giving ([], False, 0). The contract's limits and the changelog fragment name the assignments after time. Signed-off-by: Anton Volkov --- ...102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/cli_traces.py | 12 +++++---- dokimi/dokimi/repos/shell_words.py | 26 ++++++++++++++++--- dokimi/tests/test_repo_events_bash.py | 20 +++++++++++++- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 060c1f6a9..e489335a7 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options, names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options (and assignments after `time`: `time X=1 cd sub` runs the `cd` in the shell), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. ### Added diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index daf304368..57e36e0c8 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1326,7 +1326,8 @@ observations of that run. It cannot show: is not followed, so a later relative read is taken from the directory before it (an `eval`, `source` or `.` is taken as moving the shell). A command the parser does not fully follow counts as moving the shell when - its command word, after `builtin`, `command` or `time` and their options, + its command word, after `builtin`, `command` or `time` and their options + (and, after `time`, assignments: `time X=1 cd sub` runs the `cd` here), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`; not `time git add .`), and so does one in a `case` arm. `time` is that keyword only as the first word, unquoted (`X=1 time cd sub` and diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index fc86351c0..25a00e84f 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -128,7 +128,7 @@ def _walk_script(script: str, cwd: str) -> tuple[list[str], _Walk]: if not chained and walk.gated: # a cd after && or || may not have run: the next chain starts in no known directory walk.here, walk.gated = None, False walk.chain(op, chained) - if walk.in_function(_bare(part)) or walk.in_case(outer): # a definition runs nothing; a case is not followed + if walk.in_function(_bare(part)) or walk.in_case(outer, after, out): # a definition runs nothing; a case is not followed continue if op == "||" and walk.moved: walk.here = None @@ -191,15 +191,17 @@ def in_function(self, bare: str) -> bool: self.function = max(self.function + len(_GROUP_OPEN.findall(bare)) - len(_GROUP_CLOSE.findall(bare)), 0) return True - def in_case(self, outer: str) -> bool: + def in_case(self, outer: str, after: str, out: list[str]) -> bool: """Whether ``outer`` is part of a ``case ... esac``, which is not followed: an arm may run or not, so one naming a - directory builtin (as any word: ``case x in a) cd sub;; esac``) may move the shell, and its reads are not kept.""" + directory builtin (as any word: ``case x in a) cd sub;; esac``) may move the shell, and its reads are not kept. + Inside one, each ``case`` word opens another, so a miscount stays inside; its braces still open and close groups.""" words = _unquoted(outer.split()) while words[:1] and (words[0] in _KEYWORDS or words[0] == "{"): words = words[1:] - self.case += words[:1] == ["case"] - if not self.case: + if not self.case and words[:1] != ["case"]: return False + self.case += words.count("case") + self.track_groups(_bare(outer), after, out) # before lose: a backgrounded group around the case restores it if any(w in _DIRS for w in words): self.lose() self.case -= words[:1] == ["esac"] diff --git a/dokimi/dokimi/repos/shell_words.py b/dokimi/dokimi/repos/shell_words.py index 667d92c64..5b262d4f6 100644 --- a/dokimi/dokimi/repos/shell_words.py +++ b/dokimi/dokimi/repos/shell_words.py @@ -85,8 +85,8 @@ def _may_run_dir(words: list[str], timed: bool) -> bool: def _command_at(words: list[str], timed: bool) -> int | None: """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and, when ``timed``, a first - ``time``, with their options (``time -p``, ``command -p``, ``--``); None for ``command -v`` / ``-V``, which only - describes the command.""" + ``time``, with their options (``time -p``, ``command -p``, ``--``) and, after ``time``, ``NAME=value`` words; None for + ``command -v`` / ``-V``, which only describes the command.""" at = 0 while at < len(words) and _prefix(words[at], timed and at == 0): prefix, at = words[at], at + 1 @@ -96,9 +96,27 @@ def _command_at(words: list[str], timed: bool) -> int | None: at += 1 if words[at - 1] == "--": break + if prefix == "time": # the keyword times a whole simple command, its assignments too: ``time -p X=1 cd`` + at = _past_assignments(words, at) return at +def _past_assignments(words: list[str], at: int) -> int: + """The index of the first word of ``words`` from ``at`` on that is not ``NAME=value``.""" + while at < len(words) and _ASSIGN.match(words[at]): + at += 1 + return at + + +def _prefix_runs_dir(words: list[str], timed: bool) -> bool: + """Whether ``words`` start with ``builtin``, ``command`` or bash's ``time`` (``timed``) and then a directory builtin, + which the prefix runs in this shell; past ``time``, ``NAME=value`` words may come first (``time X=1 cd sub``).""" + if not _prefix(words[0], timed): + return False + at = _past_assignments(words, 1) if words[0] == "time" else 1 + return at < len(words) and words[at] in _DIRS + + def _program(word: str) -> str: """The program a word names when it is a bare name or a system path (``env``, ``/usr/bin/env``); ``""`` for a path elsewhere (``./env``), which is some program of the repository, not the one the system has.""" @@ -129,13 +147,13 @@ def _prefix(word: str, timed: bool) -> bool: def _command_words(words: list[str], timed: bool) -> tuple[list[str], bool, bool]: """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or a first ``time`` (``timed``: the - keyword) before a directory builtin (which runs it), and ``env`` with its options and assignments come before the + keyword) before a directory builtin (which runs it; ``time X=1 cd``), and ``env`` with its options and assignments come before the name and are skipped. ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" via_env = chdir = False while words: if _ASSIGN.match(words[0]): words = words[1:] - elif _prefix(words[0], timed) and len(words) > 1 and words[1] in _DIRS: # ``./command`` is a program + elif _prefix_runs_dir(words, timed): # ``./command`` is a program words = words[1:] elif _program(words[0]) == "env": words, via_env = words[1:], True diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 72ad2640f..21939d2ad 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -9,7 +9,7 @@ import pytest -from dokimi.repos.cli_traces import shell_reads_moved +from dokimi.repos.cli_traces import shell_reads_moved, shell_walk from dokimi.repos.events import convert from dokimi.repos.process_metrics import process_metrics @@ -302,6 +302,24 @@ def test_a_case_with_no_directory_builtin_keeps_the_next_read(): assert shell_reads_moved("case x in a) echo a;; esac; cat a") == (["a"], False) +def test_assignments_after_the_time_keyword_still_run_the_cd_in_this_shell(): + # bash's ``time`` times the whole simple command ``X=1 cd sub``: the cd runs here + assert shell_walk("time X=1 cd sub; cat AGENTS.md") == (["sub/AGENTS.md"], True, 0) + assert _reads(_events(_call("a", "time X=1 cd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == [] + + +def test_a_nested_case_keeps_the_outer_case_open_past_the_inner_esac(): + # the cd is still in the outer case's arm: not followed, so the cat after the case is at an unknown directory + command = "case $x in a) case $y in b) :;; esac; cd sub;; esac; cat AGENTS.md" + assert shell_walk(command) == ([], True, 1) + assert _reads(_events(_call("a", command), _result("a"))) == [] + + +def test_a_brace_opened_on_a_case_part_is_tracked_so_the_outer_redirect_hides_the_group(): + assert shell_walk("{ cat AGENTS.md; { case $x in a) :;; esac; }; } > /dev/null") == ([], False, 0) + assert shell_walk("{ cat AGENTS.md; { case $x in a) :;; esac; }; }") == (["AGENTS.md"], False, 0) + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From 7434efd49254be1fc79a6f592a48338a88eeab9e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 19:54:00 +0000 Subject: [PATCH 35/42] fix(repo): a failed one-file read adds nothing to unknown_reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles review comment 4459651043 on #369. A shell command that names one file at a directory the walker cannot follow (`cd -; cat missing.md`) added 1 to `process.unknown_reads` when it started, and its failure never took it back, although the same failure drops a known read's event. `_failed_read` now decides from the printer's own operand count also when no path was resolved, and each converter settles a command's unknown count with its result: Codex keeps it with the started command's derived reads and subtracts it on a failed one-file completion (a completion without a start adds it only when it did not fail), Copilot keeps it per call id and subtracts it on such a result, and the Claude Code Bash settlement adds it only for calls that neither were refused nor failed as one read. The contract's §4.1 `unknown_reads` row says that a failed one-file read counts none. Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/cli_traces.py | 63 +++++++++++++++---------- dokimi/dokimi/repos/events.py | 9 ++-- dokimi/tests/test_repo_cli_traces.py | 22 +++++++++ dokimi/tests/test_repo_events_bash.py | 2 + 5 files changed, 67 insertions(+), 31 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 57e36e0c8..671f7c56e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -494,7 +494,7 @@ parent's and subagents' together, so a grader need not: | `failed_commands` | commands not denied whose `rc` is neither `0` nor `null` | | `denied_commands` | commands with `denied: true` (a hook or the harness's permissions refused them) | | `guidance_read` | sorted, distinct workspace paths (normalized: `./a`, `a/../b`; not `~`) of `read` events, neither denied nor failed (`rc` 1: the file does not exist), that `guidance.globs` match. A read event is a read tool's (Claude Code's Read, Copilot's `view`) or a shell read: a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files, as §7 lists (a bare `cat`, `head`, `tail`, `nl` or `sed -n`; `cd ; cat AGENTS.md` reads `/AGENTS.md`). A command that is one such read naming one file, alone, the one command of a brace group (`{ cat AGENTS.md; }`) or after `cd` commands (`cd sub && cat AGENTS.md`; `time cd`, `command cd`, `builtin cd`, `X=1 cd`, a quoted or escaped `"cd"` or `\cd` too, not `env cd`), and fails is no read; one naming two keeps its reads, even when only one is known (`cd -; cat missing.md /ws/AGENTS.md`). Guidance seen through a pipe, `grep` or `rg` is not counted, nor a relative read after an earlier Claude Code Bash call ran a `cd` (`time cd` and `command cd` too), `pushd`, `popd`, or an `eval`, `source` or `.` (which may run one), its name quoted or escaped too (`"source" "$ENV"`, `\cd "$D"`), in its own shell, not in a subshell, pipeline or background job (the shell keeps its directory between calls, so the file is not known; a call a hook or the permissions refused ran nothing and moves nothing). That holds across agents: after the parent's `cd`, a subagent's relative reads are dropped too, and the reverse, since a subagent may share the parent's shell (not checked on a real trace; dropping a read is the safe side either way); to count those, add a `process_metrics` counter on the command text (§2), which also counts denied commands, unlike this row | -| `unknown_reads` | the relative shell reads, of calls not refused, that the trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call): they have no `read` event, so `guidance_read` may miss them; `0` when there are none | +| `unknown_reads` | the relative shell reads, of calls neither refused nor failed as one read of one file (see `guidance_read`; `cd -; cat missing.md` failing counts none), that the trace made at a directory the converter could not follow (after `source .venv/bin/activate`, `cd -`, or a `cd` in an earlier Claude Code Bash call): they have no `read` event, so `guidance_read` may miss them; `0` when there are none | | `counters` | each `process_metrics` name (§2) and its count | `process` is `null` when the trace could not be read, the agent left none (no event and no result record), or computing the metrics failed (the error is in `agent.trace_errors`). diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 25a00e84f..c5ea0b08f 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -451,7 +451,7 @@ def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: out = Transcript() messages: list[str] = [] by_id: dict[str, dict[str, Any]] = {} # Track started items by ID to update on completion - event_reads: dict[int, list[dict[str, Any]]] = {} # command event -> the read events derived from it + event_reads: dict[int, _Derived] = {} # command event -> the read events derived from it for n, line in enumerate(raw.split("\n"), 1): # a JSON string may hold U+2028: splitlines would cut it if not line.strip(): continue @@ -477,7 +477,7 @@ def codex_json(raw: str, workdir: str | None, cwd: str = "") -> Transcript: def _codex_apply(out: Transcript, rec: dict[str, Any], messages: list[str], workdir: str | None, by_id: dict[str, dict[str, Any]] | None = None, cwd: str = "", - event_reads: dict[int, list[dict[str, Any]]] | None = None) -> None: + event_reads: dict[int, _Derived] | None = None) -> None: if by_id is None: by_id = {} if event_reads is None: @@ -523,20 +523,20 @@ def _codex_key(item: dict[str, Any], script: str) -> str: def _codex_command_started(out: Transcript, item: dict[str, Any], workdir: str | None, by_id: dict[str, dict[str, Any]], cwd: str, - event_reads: dict[int, list[dict[str, Any]]]) -> None: + event_reads: dict[int, _Derived]) -> None: script = unwrap_shell(str(item.get("command") or "")) - read_events = [{"t": None, "kind": "read", "path": relative_path(path, workdir)} - for path in _counted_reads(out, script, cwd)] + paths, unknown = _counted_reads(out, script, cwd) + read_events = [{"t": None, "kind": "read", "path": relative_path(path, workdir)} for path in paths] out.events.extend(read_events) event: dict[str, Any] = {"t": None, "kind": "command", "text": script, "rc": None} - event_reads[id(event)] = read_events + event_reads[id(event)] = (read_events, unknown) by_id[_codex_key(item, script)] = event out.events.append(event) def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[str], workdir: str | None, by_id: dict[str, dict[str, Any]], cwd: str, - event_reads: dict[int, list[dict[str, Any]]]) -> None: + event_reads: dict[int, _Derived]) -> None: itype = item.get("type") if itype == "agent_message": messages.append(str(item.get("text") or "")) @@ -568,13 +568,14 @@ def _codex_item_completed(out: Transcript, item: dict[str, Any], messages: list[ def _codex_command_finished(out: Transcript, item: dict[str, Any], event: dict[str, Any], - event_reads: dict[int, list[dict[str, Any]]]) -> None: + event_reads: dict[int, _Derived]) -> None: """Update the event from item.started with completion data.""" event["rc"] = item.get("exit_code") if item.get("aggregated_output"): event["output"], event["output_truncated"] = clip_output(str(item["aggregated_output"])) - derived = event_reads.get(id(event), []) - if _failed_read(str(event["text"]), derived, event["rc"]): # drop the read events derived from it + derived, unknown = event_reads.get(id(event), ([], 0)) + if _failed_read(str(event["text"]), event["rc"]): # drop the read events derived from it, and its unknown ones + out.unknown_reads -= unknown dropped = {id(e) for e in derived} out.events[:] = [e for e in out.events if id(e) not in dropped] @@ -582,8 +583,9 @@ def _codex_command_finished(out: Transcript, item: dict[str, Any], event: dict[s def _codex_command_unstarted(out: Transcript, item: dict[str, Any], script: str, workdir: str | None, cwd: str) -> None: """Fallback for item.completed without item.started.""" - paths = _counted_reads(out, script, cwd) - if not _failed_read(script, paths, item.get("exit_code")): + paths, _, unknown = shell_walk(script, cwd) + if not _failed_read(script, item.get("exit_code")): + out.unknown_reads += unknown for path in paths: out.events.append({"t": None, "kind": "read", "path": relative_path(path, workdir)}) event = {"t": None, "kind": "command", "text": script, "rc": item.get("exit_code")} @@ -592,16 +594,21 @@ def _codex_command_unstarted(out: Transcript, item: dict[str, Any], script: str, out.events.append(event) -def _counted_reads(out: Transcript, script: str, cwd: str) -> list[str]: - """:func:`shell_reads`; ``out.unknown_reads`` counts the files it named at a directory not known.""" +_Derived = tuple[list[dict[str, Any]], int] # a command's read events, and how many files it named at a directory not known + + +def _counted_reads(out: Transcript, script: str, cwd: str) -> tuple[list[str], int]: + """:func:`shell_reads`, and the files it named at a directory not known, which ``out.unknown_reads`` counts + until the command's result says it failed (:func:`_failed_read`).""" reads, _, unknown = shell_walk(script, cwd) out.unknown_reads += unknown - return reads + return reads, unknown -def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: +def _failed_read(script: str, rc: Any) -> bool: """A non-zero exit of a command that is one part naming one file, alone or after ``cd`` parts - (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read. + (``cd sub && cat AGENTS.md``: the cd or the cat failed): that file was not read, known or not + (``cd -; cat missing``: no read, and no ``unknown_reads``). Otherwise the exit code does not say which read failed (``cat a missing`` prints ``a`` and exits 1), so the reads are kept: also when only one of the files is known (``cd -; cat missing /ws/a``). A part @@ -610,7 +617,7 @@ def _failed_read(script: str, reads: list[Any], rc: Any) -> bool: not parts: ``{ cat AGENTS.md; }`` and ``cd sub && { cat AGENTS.md; }`` are one read. """ parts = [p for p in (_ungrouped(part.text) for part in split_script(script)) if p] - if rc in (None, 0) or not reads or not all(_is_cd(p) for p in parts[:-1]): + if rc in (None, 0) or not all(_is_cd(p) for p in parts[:-1]): return False return _walk_script(script, "")[1].operands == 1 # the printer's own operands, not the reads kept of them @@ -626,7 +633,7 @@ class _CopilotRun: def __init__(self) -> None: self.start: datetime | None = None - self.by_id: dict[str, list[dict[str, Any]]] = {} + self.by_id: dict[str, _Derived] = {} # a call's events, and the files it named at a directory not known self.final = self.last = "" self.turns = 0 self.result = False # the final ``result`` record came: the agent started even with no turn @@ -654,11 +661,12 @@ def apply(self, out: Transcript, rec: dict[str, Any], t: float | None, workdir: elif kind == "assistant.message": self.message(data) elif kind == "tool.execution_start": + counted = out.unknown_reads events = _copilot_tool(data, t, workdir, cwd, out) out.events.extend(events) - self.by_id[str(data.get("toolCallId"))] = events + self.by_id[str(data.get("toolCallId"))] = (events, out.unknown_reads - counted) elif kind == "tool.execution_complete": - _copilot_result(out, self.by_id.get(str(data.get("toolCallId")), []), data) + _copilot_result(out, self.by_id.get(str(data.get("toolCallId")), ([], 0)), data) elif kind == "result": self.result = True out.is_error = rec.get("exitCode") not in (0, None) @@ -702,7 +710,7 @@ def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cw if name in _COPILOT_SHELL: command = str(obj.get("command", "")) # only bash is a POSIX script; a PowerShell command is not parsed for the files it reads - scripts = _counted_reads(out, command, cwd) if name == "bash" else [] + scripts = _counted_reads(out, command, cwd)[0] if name == "bash" else [] reads = [{"t": t, "kind": "read", "path": relative_path(p, workdir)} for p in scripts] return [*reads, {"t": t, "kind": "command", "text": command, "rc": None}] if name in _COPILOT_READ | _COPILOT_EDIT: @@ -718,8 +726,10 @@ def _copilot_tool(data: dict[str, Any], t: float | None, workdir: str | None, cw return [{"t": t, "kind": "tool", "name": name}] -def _copilot_result(out: Transcript, events: list[dict[str, Any]], data: dict[str, Any]) -> None: - """Fill in the completion; drop the reads of a failed ``view`` and of a failed one-file command.""" +def _copilot_result(out: Transcript, call: _Derived, data: dict[str, Any]) -> None: + """Fill in the completion; drop the reads of a failed ``view`` and of a failed one-file command, and take back + the files that command named at a directory not known.""" + events, unknown = call reads = [e for e in events if e["kind"] == "read"] failed = data.get("success") is False for event in events: @@ -732,11 +742,12 @@ def _copilot_result(out: Transcript, events: list[dict[str, Any]], data: dict[st code = shell.get("exitCode") if isinstance(shell, dict) else None event["rc"] = code if isinstance(code, int) else (None if data.get("success") is None else 0 if data.get("success") else 1) - failed = _failed_read(str(event["text"]), reads, event["rc"]) + failed = _failed_read(str(event["text"]), event["rc"]) result = data.get("result") or {} output = result.get("content") if isinstance(result, dict) else None if isinstance(output, str) and output: event["output"], event["output_truncated"] = clip_output(output) - if failed and reads: + if failed: + out.unknown_reads -= unknown dropped = {id(e) for e in reads} out.events[:] = [e for e in out.events if id(e) not in dropped] diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 37a4c3669..ce41ebfbe 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -179,14 +179,15 @@ def _settle_bash(out: Transcript, calls: Iterable[_BashCall]) -> None: every agent: a subagent may share the parent's shell; - else the reads get ``rc`` 0, since a failure does not say which file failed (``cat a missing``). - ``out.unknown_reads`` counts the relative reads of the calls not denied whose directory was not known: dropped - after a move, or after a directory change in the call that the walker could not follow (``cd -; cat a``). + ``out.unknown_reads`` counts the relative reads of the calls neither denied nor failed as above whose directory + was not known: dropped after a move, or after a directory change in the call that the walker could not follow + (``cd -; cat a``). """ moved = False # the shell may have left its start directory dropped: set[int] = set() for call in calls: - failed = call.result == "error" and _failed_read(call.text, call.named, 1) - if call.result != "denied": + failed = call.result == "error" and _failed_read(call.text, 1) + if call.result != "denied" and not failed: out.unknown_reads += call.unknown for read, path in zip(call.reads, call.named, strict=True): unknown = moved and not path.startswith("/") # the shell's directory is not known, nor the file diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index a5d053508..fcd6b3854 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -556,6 +556,28 @@ def test_codex_and_copilot_count_a_read_at_a_directory_not_known(): **copilot, "arguments": {"command": "cat AGENTS.md"}}}), None).unknown_reads == 0 +_UNKNOWN_ONE = "cd -; cat missing.md" # one file, named at a directory not known + + +@pytest.mark.parametrize("fmt, records", [ + ("codex-json", lambda rc: [{"type": "item.completed", "item": { + "type": "command_execution", "command": _UNKNOWN_ONE, "exit_code": rc}}]), + ("codex-json", lambda rc: [{"type": "item.started", "item": {"id": "a", "type": "command_execution", + "command": _UNKNOWN_ONE}}, + {"type": "item.completed", "item": {"id": "a", "type": "command_execution", + "command": _UNKNOWN_ONE, "exit_code": rc}}]), + ("copilot-json", lambda rc: [ + {"type": "tool.execution_start", "data": {"toolCallId": "a", "toolName": "bash", + "arguments": {"command": _UNKNOWN_ONE}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "a", "success": rc == 0, + "shellExecution": {"exitCode": rc}}}]), +], ids=["codex-completed", "codex-started", "copilot"]) +def test_a_failed_one_file_read_at_a_directory_not_known_counts_no_unknown_read(fmt, records): + # its exit says the one file it named was not printed: settled with the result, as a known read is + assert convert(fmt, _jsonl(*records(1)), None).unknown_reads == 0 + assert convert(fmt, _jsonl(*records(0)), None).unknown_reads == 1 + + @pytest.mark.parametrize("command, unknown", [ ("{ source env.sh; cat AGENTS.md; } > /dev/null", 0), # the group's output is hidden: not the agent's ("{ source env.sh; cat AGENTS.md; } | head", 0), diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 21939d2ad..1a210913b 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -360,6 +360,8 @@ def _process(*lines: dict) -> dict: @pytest.mark.parametrize("lines, unknown, guidance", [ ((_call("a", "source .venv/bin/activate && pytest"), _result("a"), _call("b", "cat AGENTS.md"), _result("b")), 1, []), ((_call("a", "cd - && cat AGENTS.md README.md"), _result("a")), 2, []), # within the call: cd - is not followed + ((_call("a", "cd -; cat missing.md"), _result("a")), 1, []), + ((_call("a", "cd -; cat missing.md"), _result("a", "No such file", True)), 0, []), # one file, failed: not read ((_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md /ws/a.md"), _result("b", "x", True)), 1, []), ((_call("a", "cd sub"), _result("a"), _call("b", "cat AGENTS.md"), _result("b", "x", True)), 0, []), # it failed ((_call("a", "cd sub"), _result("a"), _call("b", "cd -; cat AGENTS.md"), _denials("b")), 0, []), # nothing ran From 92c790d0995aeec7ff53998d91d5f9ebc5ad77ec Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 19:56:15 +0000 Subject: [PATCH 36/42] fix(repo): a leading redirection is not a cd's command word Settles review comment 4459651056 on #369. In `> /tmp/cd.log cd sub` the `cd` runs in the shell, but the command-word lookup took `>` as the command, so the move was missed and a later Bash `cat AGENTS.md` was recorded as the root file's read. `_command_words`, `_command_at` and `_prefix_runs_dir` now skip leading redirections (`>`, `>>`, `<`, `&>`, `2>err`, `{fd}>x`; the target attached or the next word), alongside the `NAME=value` words they already skip, and also after `builtin`, `command` and `time`. Such a part is still not followed (as `cd sub > log` is not), so the shell moves to a directory not known: within the call the next relative read is dropped and counted in `unknown_reads`, and so is the next call's. A redirected `cd` in a job or a pipeline still moves nothing, and `> log time cd sub` stays /usr/bin/time's child. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/shell_words.py | 44 ++++++++++++++++----------- dokimi/tests/test_repo_events_bash.py | 14 +++++++++ 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/dokimi/dokimi/repos/shell_words.py b/dokimi/dokimi/repos/shell_words.py index 5b262d4f6..58fdede05 100644 --- a/dokimi/dokimi/repos/shell_words.py +++ b/dokimi/dokimi/repos/shell_words.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """A shell command's words, for :func:`~dokimi.repos.cli_traces.shell_reads`: the command word past what comes before -it (``NAME=value``, ``builtin``, ``command``, ``time``, ``env``), and whether it runs a directory builtin.""" +it (``NAME=value``, a redirection, ``builtin``, ``command``, ``time``, ``env``), and whether it runs a directory builtin.""" from __future__ import annotations @@ -12,6 +12,8 @@ _SYSTEM_BIN = {"/bin", "/usr/bin", "/usr/local/bin"} _ASSIGN = re.compile(r"[A-Za-z_]\w*=") # a NAME=value word: a variable set for the command that follows +# a redirection word (``>``, ``2>err``, ``>log``, ``{fd}>x``): group 1 is its attached target, if any +_REDIRECT = re.compile(r"(?:\d+|&|\{\w+\})?(?:>[>|&]?|<(?:<])?)(.*)", re.S) _KEYWORDS = {"if", "then", "else", "elif", "do", "while", "until", "!"} # after these builtins the directory is not known: a directory stack, or a script that may run a ``cd`` _DIR_LOST = {"pushd", "popd", "eval", "source", "."} @@ -85,8 +87,8 @@ def _may_run_dir(words: list[str], timed: bool) -> bool: def _command_at(words: list[str], timed: bool) -> int | None: """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and, when ``timed``, a first - ``time``, with their options (``time -p``, ``command -p``, ``--``) and, after ``time``, ``NAME=value`` words; None for - ``command -v`` / ``-V``, which only describes the command.""" + ``time``, with their options (``time -p``, ``command -p``, ``--``), redirections and, after ``time``, ``NAME=value`` + words; None for ``command -v`` / ``-V``, which only describes the command.""" at = 0 while at < len(words) and _prefix(words[at], timed and at == 0): prefix, at = words[at], at + 1 @@ -96,24 +98,31 @@ def _command_at(words: list[str], timed: bool) -> int | None: at += 1 if words[at - 1] == "--": break - if prefix == "time": # the keyword times a whole simple command, its assignments too: ``time -p X=1 cd`` - at = _past_assignments(words, at) + # the keyword times a whole simple command, its assignments too: ``time -p X=1 cd`` + at = _past_redirections(words, at, prefix == "time") return at -def _past_assignments(words: list[str], at: int) -> int: - """The index of the first word of ``words`` from ``at`` on that is not ``NAME=value``.""" - while at < len(words) and _ASSIGN.match(words[at]): - at += 1 - return at +def _past_redirections(words: list[str], at: int, assignments: bool = True) -> int: + """The index of the first word of ``words`` from ``at`` on that is neither a redirection (``> log``, ``2>err``, + which is no command word) nor, when ``assignments``, ``NAME=value``.""" + while at < len(words): + if assignments and _ASSIGN.match(words[at]): + at += 1 + elif m := _REDIRECT.fullmatch(words[at]): + at += 1 if m.group(1) else 2 # its target is attached, or the next word + else: + break + return min(at, len(words)) def _prefix_runs_dir(words: list[str], timed: bool) -> bool: """Whether ``words`` start with ``builtin``, ``command`` or bash's ``time`` (``timed``) and then a directory builtin, - which the prefix runs in this shell; past ``time``, ``NAME=value`` words may come first (``time X=1 cd sub``).""" + which the prefix runs in this shell; past ``time``, ``NAME=value`` words may come first (``time X=1 cd sub``), and + redirections past any of them.""" if not _prefix(words[0], timed): return False - at = _past_assignments(words, 1) if words[0] == "time" else 1 + at = _past_redirections(words, 1, words[0] == "time") return at < len(words) and words[at] in _DIRS @@ -146,13 +155,14 @@ def _prefix(word: str, timed: bool) -> bool: def _command_words(words: list[str], timed: bool) -> tuple[list[str], bool, bool]: """The words from the command's name on; whether ``env`` runs it; and whether that ``env`` changes - directory (``-C`` / ``--chdir``). ``NAME=value`` words, ``builtin``, ``command`` or a first ``time`` (``timed``: the - keyword) before a directory builtin (which runs it; ``time X=1 cd``), and ``env`` with its options and assignments come before the - name and are skipped. ``words`` are unquoted: ``shlex.split``'s, or :func:`_unquoted`'s for a part shlex does not split.""" + directory (``-C`` / ``--chdir``). ``NAME=value`` words, redirections (``> log cd sub``), ``builtin``, ``command`` or + a first ``time`` (``timed``: the keyword) before a directory builtin (which runs it; ``time X=1 cd``), and ``env`` + with its options and assignments come before the name and are skipped. ``words`` are unquoted: ``shlex.split``'s, + or :func:`_unquoted`'s for a part shlex does not split.""" via_env = chdir = False while words: - if _ASSIGN.match(words[0]): - words = words[1:] + if _ASSIGN.match(words[0]) or _REDIRECT.fullmatch(words[0]): + words = words[_past_redirections(words, 0):] elif _prefix_runs_dir(words, timed): # ``./command`` is a program words = words[1:] elif _program(words[0]) == "env": diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 1a210913b..e175dbc7b 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -320,6 +320,20 @@ def test_a_brace_opened_on_a_case_part_is_tracked_so_the_outer_redirect_hides_th assert shell_walk("{ cat AGENTS.md; { case $x in a) :;; esac; }; }") == (["AGENTS.md"], False, 0) +@pytest.mark.parametrize("first", ["> /tmp/cd.log cd sub", ">/tmp/cd.log cd sub", "log cd sub", + "X=1 > log cd sub", "command > log cd sub", "command -p > log cd sub", + "> log source env.sh"]) +def test_a_leading_redirection_is_no_command_word_so_the_cd_moves_the_shell(first): + # a redirect is not followed (as ``cd sub > log``): the shell moved, to a directory not known here + assert shell_walk(f"{first}; cat AGENTS.md") == ([], True, 1) + assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == [] + + +def test_a_leading_redirection_of_a_cd_in_a_job_or_a_pipeline_moves_no_shell(): + assert shell_walk("> log cd sub & cat AGENTS.md") == (["AGENTS.md"], False, 0) + assert shell_walk("> log cd sub | cat; cat AGENTS.md") == (["AGENTS.md"], False, 0) + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From e8c06f0e187a0590a6d08fbcadb80d7bd4d826f7 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 20:14:18 +0000 Subject: [PATCH 37/42] fix(repo): a redirection before a prefix's options hides no cd Settles review comment 4459886479 on #369. `_command_at` skipped a `builtin`, `command` or `time` prefix's options only before the redirections that follow it, so in `command > /tmp/cd.log -p cd sub` the command word came out as `-p`: the cd was missed, and a later relative Bash read was recorded at the start directory. `_past_prefix` now skips the prefix's options, redirections and, past `time`, `NAME=value` words in one loop, in any order, until the command word. Such a cd still loses the directory and moves the shell, as a redirected cd does since the previous commit; `command > log -v cd` still runs nothing. Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/shell_words.py | 29 ++++++++++++++++++--------- dokimi/tests/test_repo_events_bash.py | 12 +++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/repos/shell_words.py b/dokimi/dokimi/repos/shell_words.py index 58fdede05..5eec955b6 100644 --- a/dokimi/dokimi/repos/shell_words.py +++ b/dokimi/dokimi/repos/shell_words.py @@ -89,17 +89,26 @@ def _command_at(words: list[str], timed: bool) -> int | None: """The index of the command word in unquoted ``words``, past ``builtin``, ``command`` and, when ``timed``, a first ``time``, with their options (``time -p``, ``command -p``, ``--``), redirections and, after ``time``, ``NAME=value`` words; None for ``command -v`` / ``-V``, which only describes the command.""" - at = 0 - while at < len(words) and _prefix(words[at], timed and at == 0): - prefix, at = words[at], at + 1 - while at < len(words) and words[at].startswith("-") and words[at] != "-": - if prefix == "command" and set(words[at]) & {"v", "V"}: + at: int | None = 0 + while at is not None and at < len(words) and _prefix(words[at], timed and at == 0): + at = _past_prefix(words, at) + return at + + +def _past_prefix(words: list[str], at: int) -> int | None: + """The index past the prefix ``words[at]`` and the options, redirections and, after ``time``, ``NAME=value`` words + that follow it in any order (``command > log -p cd``, ``time -p X=1 cd``); None for ``command -v`` / ``-V``.""" + prefix, at, options = words[at], at + 1, True + while at < len(words): + word = words[at] + if options and word.startswith("-") and word != "-": + if prefix == "command" and set(word) & {"v", "V"}: return None - at += 1 - if words[at - 1] == "--": - break - # the keyword times a whole simple command, its assignments too: ``time -p X=1 cd`` - at = _past_redirections(words, at, prefix == "time") + at, options = at + 1, word != "--" # past ``--`` a word is no option + elif (past := _past_redirections(words, at, prefix == "time")) > at: + at = past # the keyword times a whole simple command, its assignments too + else: + break return at diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index e175dbc7b..a07edf1ad 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -334,6 +334,18 @@ def test_a_leading_redirection_of_a_cd_in_a_job_or_a_pipeline_moves_no_shell(): assert shell_walk("> log cd sub | cat; cat AGENTS.md") == (["AGENTS.md"], False, 0) +@pytest.mark.parametrize("first", ["command > /tmp/cd.log -p cd sub", "time > log -p cd sub", + "builtin > log -- cd sub", "time -p > log X=1 cd sub"]) +def test_a_redirection_before_a_prefixs_options_still_finds_the_cd(first): + # the prefix's options, redirections and (past ``time``) assignments come in any order before the command word + assert shell_walk(f"{first}; cat AGENTS.md") == ([], True, 1) + assert _reads(_events(_call("a", first), _result("a"), _call("b", "cat AGENTS.md"), _result("b"))) == [] + + +def test_a_redirection_before_command_v_still_runs_nothing(): + assert shell_walk("command > log -v cd; cat AGENTS.md") == (["AGENTS.md"], False, 0) + + def test_a_parents_cd_drops_a_subagents_later_relative_read(): # a subagent may share the parent's shell, so its relative read after the parent's cd is not known events = _events(_call("s1", "cat AGENTS.md", agent="t1"), _result("s1"), From 2f5dfd3044c418b371e1eaf1cc3c41a2029d4251 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 4 Nov 2026 11:02:38 +0100 Subject: [PATCH 38/42] docs(repo): first_useful.steps counts a Claude Bash read as one more step A derived read event is one more event, so a run that prints a file through Bash before its first useful result shows one step more. The changelog says so and a test pins it. Signed-off-by: Anton Volkov --- ...61102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/tests/test_repo_useful.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index e489335a7..0d214d3f5 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options (and assignments after `time`: `time X=1 cd sub` runs the `cd` in the shell), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options (and assignments after `time`: `time X=1 cd sub` runs the `cd` in the shell), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `first_useful.steps` counts events, so a Claude Code run that prints a file through Bash before its first useful result shows one step more than before (`cat AGENTS.md`, then a passing `pytest`: 3, was 2), and `issued_s` is unchanged; `repo compare` pairs steps with batches run before this change on that basis. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. ### Added diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 72321fee0..54ddffcf4 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -254,6 +254,23 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): } +def test_a_claude_bash_read_is_one_more_step_before_the_first_useful_command(): + def call(i: str, command: str, t: str) -> list[dict]: + use = {"type": "tool_use", "id": i, "name": "Bash", "input": {"command": command}} + res = {"type": "tool_result", "tool_use_id": i, "is_error": False, "content": "ok"} + return [ + {"type": "assistant", "timestamp": t, "message": {"content": [use]}}, + {"type": "user", "timestamp": t, "message": {"content": [res]}}, + ] + + lines = call("a", "cat AGENTS.md", "2026-10-31T10:00:01.000Z") + lines += call("b", "pytest tests/test_x.py", "2026-10-31T10:00:05.000Z") + raw = "\n".join(json.dumps(x) for x in lines) + events = convert("claude-stream-json", raw, "/ws").events + assert [e["kind"] for e in events] == ["read", "command", "command"] + assert first_useful(parse_useful({"command": TEST_CMD}), events)["steps"] == 3 + + # -- report and compare rows ------------------------------------------------------------------------------------- From 21e0690303abfa1ded4400cd441f8eef03bfb9af Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 4 Nov 2026 03:58:32 -0800 Subject: [PATCH 39/42] fix(repo): the bash= probe may run only the cat it names repo reach runs the shell on the host, so a probed CLAUDE.md could drive any command. claude-code's bash mode adds a PreToolUse hook that blocks all but DOKIMI_REACH_COMMAND, and fails closed. Signed-off-by: Alexander Rybkin --- ...2_120100_antonwolfy_reach-bash-scenario.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 5 +- dokimi/dokimi/repos/profiles/claude-code.yaml | 11 ++++- dokimi/dokimi/repos/reach.py | 23 +++++++-- dokimi/dokimi/schemas/harness_profile.v1.json | 2 +- dokimi/tests/test_batch_ledger_golden.py | 2 +- dokimi/tests/test_repo_reach_bash.py | 48 +++++++++++++++++-- 7 files changed, 81 insertions(+), 12 deletions(-) diff --git a/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md index 5ea380c2c..b2ab78328 100644 --- a/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md +++ b/dokimi/changelog.d/20261102_120100_antonwolfy_reach-bash-scenario.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **A `bash=` reach scenario** — `repo reach --scenarios bash` runs, for each guidance directory, a scenario started at the root with the profile's new `bash` tool mode (`claude-code`: `--tools Bash`) that tells the agent to `cat /REACH_PROBE.txt` (the path shell-quoted, so a space or `;` in a directory name stays one operand) and then list tokens: it shows whether a shell read of a nested file loads that directory's guidance, as `read=` shows for the Read tool. The default stays `root,cwd,read`, so a probe costs no more unless asked; `--dry-run` lists the bash scenarios and `--only bash=` picks one. A profile without a `bash` tool mode (`codex`, `copilot`) asked for `--scenarios bash` is a usage error (exit 2) naming the profile. `harness_profile.v1` accepts an optional `bash` key under `tools` (not under `delegation.none`, so `repo run --tools bash --delegation none` is refused); the receipt treats a `bash=` run as it does `read=`. +- **A `bash=` reach scenario** — `repo reach --scenarios bash` runs, for each guidance directory, a scenario started at the root with the profile's new `bash` tool mode (`claude-code`: `--tools Bash`) that tells the agent to `cat /REACH_PROBE.txt` (the path shell-quoted, so a space or `;` in a directory name stays one operand) and then list tokens: it shows whether a shell read of a nested file loads that directory's guidance, as `read=` shows for the Read tool. The default stays `root,cwd,read`, so a probe costs no more unless asked; `--dry-run` lists the bash scenarios and `--only bash=` picks one. A profile without a `bash` tool mode (`codex`, `copilot`) asked for `--scenarios bash` is a usage error (exit 2) naming the profile. `harness_profile.v1` accepts an optional `bash` key under `tools` (not under `delegation.none`, so `repo run --tools bash --delegation none` is refused); the receipt treats a `bash=` run as it does `read=`. The scenario runs a shell on the host, with the harness's credentials and network, so `claude-code`'s `bash` mode adds a PreToolUse hook that blocks every Bash call but the exact `cat` that `repo reach` names in `DOKIMI_REACH_COMMAND`, and blocks every call when the hook itself fails: a `CLAUDE.md` in the probed repository cannot make the probe run anything else. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 0fb651612..2a03f7367 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1142,7 +1142,10 @@ Protocol: read of a file in `` loads that directory's guidance. Only with `--scenarios bash` (the default is `root,cwd,read`); a profile without a `bash` tool mode (Codex and Copilot, whose shell commands are already their reads) is a - usage error (exit 2) naming the profile. + usage error (exit 2) naming the profile. The shell runs on the host with + the harness's credentials, so the mode must refuse every other command: + Claude Code's gets the one `cat` in `DOKIMI_REACH_COMMAND`, and a + PreToolUse hook blocks any other call (and every call if the hook fails). - `shadow=.` and `shadow=`, only with `--scenarios shadow` (the default is `root,cwd,read`): the profile's `guidance.shadow_file` (§6) is written into the workspace for that one scenario, seeded with its own diff --git a/dokimi/dokimi/repos/profiles/claude-code.yaml b/dokimi/dokimi/repos/profiles/claude-code.yaml index c7d7c9f52..58039a1dd 100644 --- a/dokimi/dokimi/repos/profiles/claude-code.yaml +++ b/dokimi/dokimi/repos/profiles/claude-code.yaml @@ -28,7 +28,16 @@ invoke: tools: none: ["--tools", ""] read: ["--tools", "Read"] - bash: ["--tools", "Bash"] # repo reach --scenarios bash: what a shell read of a nested file loads + # repo reach --scenarios bash: what a shell read of a nested file loads. The probe runs on the host, + # so a hook refuses every command but the one reach names (a hook that fails refuses it too). + bash: + - "--tools" + - "Bash" + - "--settings" + - >- + {"hooks": {"PreToolUse": [{"matcher": "Bash", "hooks": [{"type": "command", + "command": "python3 -c 'import json, os, sys; sys.exit(json.load(sys.stdin)[\"tool_input\"][\"command\"] + != os.environ[\"DOKIMI_REACH_COMMAND\"])' || exit 2"}]}]}} all: ["--disallowedTools", "WebFetch", "WebSearch"] # repo skill-reach: with no tools Claude Code lists no skills, so only the Skill tool is left. skills: ["--tools", "Skill"] diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index 513291e72..a08769975 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -89,7 +89,9 @@ "and nothing else. Report only what is already in your context; do not search for " "files. If there are none, answer NONE.") ASK_AFTER_READ = "First use the Read tool to read the file {path}, and read no other file. Then: " -ASK_AFTER_BASH = "First use the Bash tool to run `cat {path}`, and run no other command. Then: " +ASK_AFTER_BASH = "First use the Bash tool to run `{command}`, and run no other command. Then: " +# The one shell command a bash scenario's harness may run; the profile's bash tool mode must refuse any other. +ONLY_COMMAND_ENV = "DOKIMI_REACH_COMMAND" class ReachError(RuntimeError): @@ -120,12 +122,17 @@ class Scenario: read: str | None = None # file the agent is told to read first (with ``cat`` in the bash tool mode) shadow: str | None = None # the shadow file written for this scenario only + @property + def command(self) -> str | None: + """The shell command a ``bash`` scenario names, the only one its harness may run.""" + return f"cat {shell_word(self.read)}" if self.read and self.tools == "bash" else None + @property def prompt(self) -> str: if not self.read: return ASK - if self.tools == "bash": - return ASK_AFTER_BASH.format(path=shell_word(self.read)) + ASK + if self.command: + return ASK_AFTER_BASH.format(command=self.command) + ASK return ASK_AFTER_READ.format(path=self.read) + ASK @@ -607,7 +614,7 @@ def _run_one(profile: HarnessProfile, scenario: Scenario, model: str, workspace: "read": scenario.read, "returncode": None, "error": None, "cost_usd": None, "num_turns": None, "reported": [], "unknown": [], "prose": None} try: - proc = process.run(argv, cwd=workspace / scenario.cwd, env=profile.env(config_dir), + proc = process.run(argv, cwd=workspace / scenario.cwd, env=scenario_env(profile, scenario, config_dir), timeout=timeout_s, capture_limit=None) # the trace is parsed whole except OSError as exc: row.update(error=f"{type(exc).__name__}: {exc}", _stdout="") @@ -627,6 +634,14 @@ def _run_one(profile: HarnessProfile, scenario: Scenario, model: str, workspace: return transcript, row +def scenario_env(profile: HarnessProfile, scenario: Scenario, config_dir: Path) -> process.Env: + """The profile's child environment, plus, in a bash scenario, the one command its shell may run.""" + env = profile.env(config_dir) + if not scenario.command: + return env + return process.Env(env.allow, {**env.set, ONLY_COMMAND_ENV: scenario.command}) + + def _matrix(records: list[dict[str, Any]]) -> dict[str, dict[str, str]]: """file -> scenario -> class, for the text report.""" matrix: dict[str, dict[str, str]] = {} diff --git a/dokimi/dokimi/schemas/harness_profile.v1.json b/dokimi/dokimi/schemas/harness_profile.v1.json index bd76e2f3f..51aebd3bc 100644 --- a/dokimi/dokimi/schemas/harness_profile.v1.json +++ b/dokimi/dokimi/schemas/harness_profile.v1.json @@ -19,7 +19,7 @@ "properties": { "none": {"type": "array", "items": {"type": "string"}}, "read": {"type": "array", "items": {"type": "string"}}, - "bash": {"type": "array", "items": {"type": "string"}, "description": "The shell tool alone: `repo reach --scenarios bash`. A profile whose shell commands already give its read events (Codex, Copilot) needs none."}, + "bash": {"type": "array", "items": {"type": "string"}, "description": "The shell tool alone: `repo reach --scenarios bash`, on the host, so it must refuse every command but the one in DOKIMI_REACH_COMMAND. A profile whose shell commands already give its read events (Codex, Copilot) needs none."}, "all": {"type": "array", "items": {"type": "string"}}, "skills": {"type": "array", "items": {"type": "string"}, "description": "For `repo skill-reach`: the agent sees its skill list but can run nothing (Claude Code needs its Skill tool for the list to appear)."} } diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index 797753482..ac45bf111 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -24,7 +24,7 @@ GOLDEN_CLAUDE_CODE_IDS = { ("all", "native"): "e6eabec71f42", ("all", "none"): "2324b5c59f99", - ("bash", "native"): "ebecf5b62ae8", + ("bash", "native"): "c8f23b20ef75", ("none", "native"): "7d8aa618677b", ("none", "none"): "7d8aa618677b", ("read", "native"): "e9dbd28050f2", diff --git a/dokimi/tests/test_repo_reach_bash.py b/dokimi/tests/test_repo_reach_bash.py index dd7c9f6d9..6e0aae650 100644 --- a/dokimi/tests/test_repo_reach_bash.py +++ b/dokimi/tests/test_repo_reach_bash.py @@ -12,7 +12,10 @@ from __future__ import annotations import json +import os import re +import shutil +import subprocess import sys from pathlib import Path @@ -23,7 +26,8 @@ from dokimi.repos import WorkTree, load_manifest from dokimi.repos.cli_traces import shell_reads from dokimi.repos.harness_profile import ProfileError, load_profile -from dokimi.repos.reach import ASK, ASK_AFTER_BASH, ASK_AFTER_READ, ReachError, run_reach, scenarios +from dokimi.repos.reach import (ASK, ASK_AFTER_BASH, ASK_AFTER_READ, ONLY_COMMAND_ENV, ReachError, run_reach, + scenario_env, scenarios) from dokimi.repos.receipt import build_receipt from tests.test_repo_reach import _write from tests.test_repo_receipt import BUDGET, _record, _sources, _tree @@ -108,7 +112,7 @@ def test_bash_scenarios_cat_each_probe_file_from_the_root(): ("bash=a", "", "bash", "a/REACH_PROBE.txt"), ("bash=sub", "", "bash", "sub/REACH_PROBE.txt")] assert got[1].prompt == ("First use the Bash tool to run `cat sub/REACH_PROBE.txt`, and run no other command. " "Then: " + ASK) - assert got[1].prompt == ASK_AFTER_BASH.format(path="sub/REACH_PROBE.txt") + ASK + assert got[1].prompt == ASK_AFTER_BASH.format(command="cat sub/REACH_PROBE.txt") + ASK assert not [s for s in scenarios(files) if s.tools == "bash"] # the default kinds run no bash scenario @@ -131,11 +135,49 @@ def test_a_fake_bash_run_reads_the_probe_of_a_directory_with_a_space(tmp_path, m def test_the_claude_code_profile_runs_the_bash_mode_with_the_bash_tool_alone(): profile = load_profile("claude-code") - assert profile.argv(prompt="p", model="m", tools="bash")[-2:] == ["--tools", "Bash"] + assert profile.argv(prompt="p", model="m", tools="bash")[-4:-1] == ["--tools", "Bash", "--settings"] with pytest.raises(ProfileError, match="lists no arguments for tool mode 'bash'"): # a reach mode, not a run's profile.argv(prompt="p", model="m", tools="bash", delegation="none") +def _gate(command: str | None, call: str) -> int: + """Run the claude-code bash mode's PreToolUse hook on a Bash call, as Claude Code does, with the scenario env.""" + argv = load_profile("claude-code").argv(prompt="p", model="m", tools="bash") + hooks = json.loads(argv[argv.index("--settings") + 1])["hooks"]["PreToolUse"] + assert [h["matcher"] for h in hooks] == ["Bash"] + env = {k: v for k, v in os.environ.items() if k != ONLY_COMMAND_ENV} + env.update({ONLY_COMMAND_ENV: command} if command is not None else {}) + stdin = json.dumps({"tool_name": "Bash", "tool_input": {"command": call}}) + return subprocess.run(["sh", "-c", hooks[0]["hooks"][0]["command"]], input=stdin, env=env, text=True, + capture_output=True, timeout=60).returncode + + +needs_sh = pytest.mark.skipif(not (shutil.which("sh") and shutil.which("python3")), reason="the hook needs sh, python3") +PROBE = scenarios(["my docs/AGENTS.md"], ("bash",))[0] + + +@needs_sh +@pytest.mark.parametrize("call", ["curl x", "cat a && curl x", "cat a; id", "id", "env", + f"{PROBE.command} && id", f"{PROBE.command}; id", f"{PROBE.command} | head -1", + f"{PROBE.command}\nid", f"cat {PROBE.read}", ""]) +def test_the_claude_code_bash_mode_refuses_any_other_command(call): + assert _gate(PROBE.command, call) == 2 # exit 2: Claude Code blocks the call, in bypassPermissions too + + +@needs_sh +def test_the_claude_code_bash_mode_admits_the_probe_cat_and_fails_closed_without_one(): + assert PROBE.command == "cat 'my docs/REACH_PROBE.txt'" + assert _gate(PROBE.command, PROBE.command) == 0 + assert _gate(None, PROBE.command) == 2 # no command named: the hook errors, and refuses + + +def test_only_a_bash_scenario_names_its_command_to_the_harness(tmp_path): + profile = load_profile("claude-code") + bash, read = scenarios(["sub/AGENTS.md"], ("bash", "read"))[::-1] + assert scenario_env(profile, bash, tmp_path).set[ONLY_COMMAND_ENV] == "cat sub/REACH_PROBE.txt" == bash.command + assert ONLY_COMMAND_ENV not in scenario_env(profile, read, tmp_path).set and read.command is None + + @pytest.mark.parametrize("harness", ["codex", "copilot"]) def test_a_profile_without_a_bash_mode_is_a_usage_error(tmp_path, capsys, harness): root, _ = _setup(tmp_path) From 5aab99f980c5b1bc9b618cd805105bbb41b7937f Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 4 Nov 2026 04:03:43 -0800 Subject: [PATCH 40/42] refactor(repo): keep repo.py and test_repo_useful.py under the growth caps The reach flag check moves to reach.check_flags; the Bash first_useful test joins test_repo_events_bash.py. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/repo.py | 19 ++----------------- dokimi/dokimi/repos/reach.py | 13 +++++++++++++ dokimi/tests/test_repo_events_bash.py | 19 +++++++++++++++++++ dokimi/tests/test_repo_useful.py | 17 ----------------- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index bcc42435c..baef9a5be 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -7,7 +7,6 @@ import argparse import json -import math import re import sys from pathlib import Path @@ -274,20 +273,6 @@ def cmd_repo_fetch(args: argparse.Namespace) -> None: print(f"{tree.sha}\t{tree.git_dir}") -def _check_reach_flags(args: argparse.Namespace, kinds: tuple[str, ...]) -> None: - """Refuse a flag that would drop a scenario kind silently, or spend a budget that is not one.""" - from dokimi.repos.reach import KINDS, ReachError - - # A typo would otherwise drop a scenario kind silently and still spend the budget. - unknown_kinds = sorted(set(kinds) - set(KINDS)) - if unknown_kinds: - raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: {', '.join(KINDS)})") - if not math.isfinite(args.budget) or args.budget <= 0: - raise ReachError(f"--budget must be > 0, got {args.budget:g}") - if args.model is None: - raise ReachError("--model is required to probe (leave it out only with --from)") - - # The probe's flags, unset by default: --from refuses any of them given, even at its default value. _PROBE_ONLY = {"only": None, "salt": None, "work_dir": None, "keep": False, "bin": None, "dry_run": False, "model": None, "scenarios": None, "max_turns": None, "budget": None, "timeout": None, "out": None} @@ -443,12 +428,12 @@ def cmd_repo_reach(args: argparse.Namespace) -> None: from dokimi.repos import RepoSource, SourceError, open_source from dokimi.repos.harness_profile import ProfileError, load_profile from dokimi.repos.source import SourceArgumentError - from dokimi.repos.reach import ReachError, check_kinds, check_shadow, run_reach, save_reach + from dokimi.repos.reach import ReachError, check_flags, check_kinds, check_shadow, run_reach, save_reach kinds = tuple(k.strip() for k in args.scenarios.split(",") if k.strip()) only = tuple(args.only or ()) try: # the arguments as typed: a usage error - _check_reach_flags(args, kinds) + check_flags(kinds, args.budget, args.model) profile = load_profile(args.harness) check_kinds(profile, kinds) check_shadow(profile, kinds) diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index a08769975..d7f6c91b7 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -42,6 +42,7 @@ from __future__ import annotations import hashlib +import math import os import re import shlex @@ -98,6 +99,18 @@ class ReachError(RuntimeError): """The probe cannot run safely (for example a guidance file above the workspace).""" +def check_flags(kinds: tuple[str, ...], budget: float, model: str | None) -> None: + """Refuse a flag that would drop a scenario kind silently, or spend a budget that is not one.""" + # A typo would otherwise drop a scenario kind silently and still spend the budget. + unknown_kinds = sorted(set(kinds) - set(KINDS)) + if unknown_kinds: + raise ReachError(f"unknown --scenarios kind(s) {', '.join(unknown_kinds)} (known: {', '.join(KINDS)})") + if not math.isfinite(budget) or budget <= 0: + raise ReachError(f"--budget must be > 0, got {budget:g}") + if model is None: + raise ReachError("--model is required to probe (leave it out only with --from)") + + def check_kinds(profile: HarnessProfile, kinds: tuple[str, ...]) -> None: """Refuse a ``bash`` scenario for a profile without a ``bash`` tool mode, before any scenario runs.""" if "bash" in kinds and "bash" not in profile.tools: diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index a07edf1ad..88cf5cc1f 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -12,6 +12,8 @@ from dokimi.repos.cli_traces import shell_reads_moved, shell_walk from dokimi.repos.events import convert from dokimi.repos.process_metrics import process_metrics +from dokimi.repos.useful import first_useful +from dokimi.repos.useful_spec import parse_useful WS = "/ws" @@ -408,3 +410,20 @@ def test_guidance_printed_through_bash_is_guidance_read(): got = process_metrics(events, lambda p: p.endswith("AGENTS.md")) assert got["guidance_read"] == ["benchmarks/AGENTS.md"] assert got["commands"] == 3 and got["failed_commands"] == 1 + + +def test_a_claude_bash_read_is_one_more_step_before_the_first_useful_command(): + def call(i: str, command: str, t: str) -> list[dict]: + use = {"type": "tool_use", "id": i, "name": "Bash", "input": {"command": command}} + res = {"type": "tool_result", "tool_use_id": i, "is_error": False, "content": "ok"} + return [ + {"type": "assistant", "timestamp": t, "message": {"content": [use]}}, + {"type": "user", "timestamp": t, "message": {"content": [res]}}, + ] + + lines = call("a", "cat AGENTS.md", "2026-10-31T10:00:01.000Z") + lines += call("b", "pytest tests/test_x.py", "2026-10-31T10:00:05.000Z") + raw = "\n".join(json.dumps(x) for x in lines) + events = convert("claude-stream-json", raw, "/ws").events + assert [e["kind"] for e in events] == ["read", "command", "command"] + assert first_useful(parse_useful({"command": r"pytest tests/test_x\.py"}), events)["steps"] == 3 diff --git a/dokimi/tests/test_repo_useful.py b/dokimi/tests/test_repo_useful.py index 54ddffcf4..72321fee0 100644 --- a/dokimi/tests/test_repo_useful.py +++ b/dokimi/tests/test_repo_useful.py @@ -254,23 +254,6 @@ def test_the_hit_is_timed_from_the_claude_stream_it_came_from(): } -def test_a_claude_bash_read_is_one_more_step_before_the_first_useful_command(): - def call(i: str, command: str, t: str) -> list[dict]: - use = {"type": "tool_use", "id": i, "name": "Bash", "input": {"command": command}} - res = {"type": "tool_result", "tool_use_id": i, "is_error": False, "content": "ok"} - return [ - {"type": "assistant", "timestamp": t, "message": {"content": [use]}}, - {"type": "user", "timestamp": t, "message": {"content": [res]}}, - ] - - lines = call("a", "cat AGENTS.md", "2026-10-31T10:00:01.000Z") - lines += call("b", "pytest tests/test_x.py", "2026-10-31T10:00:05.000Z") - raw = "\n".join(json.dumps(x) for x in lines) - events = convert("claude-stream-json", raw, "/ws").events - assert [e["kind"] for e in events] == ["read", "command", "command"] - assert first_useful(parse_useful({"command": TEST_CMD}), events)["steps"] == 3 - - # -- report and compare rows ------------------------------------------------------------------------------------- From 1751a053ecf87840a6f699c716be16c4cf519aba Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 4 Nov 2026 04:33:12 -0800 Subject: [PATCH 41/42] fix(repo): a negated printer is no read `! cat AGENTS.md` of a missing file exits 0, so it was read with rc 0; its exit no longer says whether the file was printed, so it gives no read in any harness. Signed-off-by: Alexander Rybkin --- .../20261102_120000_antonwolfy_claude-bash-reads.md | 2 +- dokimi/dokimi/repos/cli_traces.py | 7 ++++--- dokimi/tests/test_repo_events_bash.py | 7 +++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md index 0d214d3f5..6579adf90 100644 --- a/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md +++ b/dokimi/changelog.d/20261102_120000_antonwolfy_claude-bash-reads.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 ### Changed -- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options (and assignments after `time`: `time X=1 cd sub` runs the `cd` in the shell), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `first_useful.steps` counts events, so a Claude Code run that prints a file through Bash before its first useful result shows one step more than before (`cat AGENTS.md`, then a passing `pytest`: 3, was 2), and `issued_s` is unchanged; `repo compare` pairs steps with batches run before this change on that basis. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. +- **A Claude Code `Bash` call that prints a file is a read, as a Codex or Copilot shell command is** — the `claude-stream-json` converter now gives a `read` event, before the `command` event, for each file a Bash command only prints (a bare `cat`, `head`, `tail`, `nl` or `sed -n`, after any `cd` in the same command: `cd benchmarks; cat AGENTS.md` reads `benchmarks/AGENTS.md`). So `result.json` `process.guidance_read` now counts guidance printed through Bash, where it counted only the Read tool before, and means the same for the three harnesses; `tool_calls` counts the new read events too, a `cat` of an installed skill's `SKILL.md` counts the skill as used, and a `cat` of a file a `patch:` arm added counts in `uptake.reads`. A command that is one read of one file and fails (`cat AGENTS.md` where there is none), alone, in a brace group (`{ cat AGENTS.md; }`) or after a `cd` (`cd sub && cat AGENTS.md`, `time cd`, `command cd`, `builtin cd` or `X=1 cd` too, which Codex and Copilot commands counted as a read before), gives no read, but one naming two files keeps its reads for all three, even when only one of them is known (`cd -; cat missing.md /ws/AGENTS.md`, whose read Codex and Copilot commands dropped before); a refused one gives denied reads; a pipe, `grep` or `rg` gives none, nor does a negated read (`! cat AGENTS.md`), whose exit says nothing of the file. Claude Code keeps the shell's directory between Bash calls, so after a call that ran a `cd`, `pushd`, or an `eval`, `source` or `.` (which may run a `cd`) in that shell (not in a subshell, pipeline or background job: `cd sub &` moves nothing, nor does a refused call), later relative reads are not recorded (absolute ones are), the parent's and every subagent's alike, since a subagent may share the parent's shell. For all three harnesses, a backgrounded or piped `cd`, `pushd`, `eval`, `source` or `.` no longer loses the directory for the rest of its command, a later `||` branch included (`source env.sh & cat AGENTS.md` and `source env.sh & false || cat AGENTS.md` read `AGENTS.md`), a `cd` inside `$( )`, backticks or `( ... )` no longer moves the shell (`(true; cd sub); cat AGENTS.md` reads `AGENTS.md`), a brace group is followed through the `$( )` or nested groups in it (`{ cd $(pwd); } | cat` moves nothing), and a directory builtin whose name is quoted or escaped is that builtin (`"source" "$ENV"` and `\cd "$D"` lose the directory, and `"cd" sub && cat AGENTS.md` that fails gives no read), and a command the parser does not fully follow whose command word, after `builtin`, `command` or `time` and their options (and assignments after `time`: `time X=1 cd sub` runs the `cd` in the shell), names a directory builtin (`X="a b" cd "$D"`, `time -p cd sub`, `command -p cd sub`, `builtin -- cd sub`; not `time git add .` or `command -v cd`) counts as moving the shell, as does one in a `case` arm, while a `time` after an assignment or quoted or escaped (`X=1 time cd sub`, `\time cd sub`) runs /usr/bin/time and moves nothing. In `repo reach`, a Bash read of a guidance file now makes it `fetched`. A read that was refused or failed, by any tool (the Read tool's too, before this change), no longer makes a file `fetched` in `repo reach` or its receipt's `states.accessed` true, nor counts a skill whose `SKILL.md` it named as used; nor does a refused or failed `Skill` tool call (a Claude Code `Skill` event now carries `rc`, `1` when the call failed), and a failed read of a file a `patch:` arm added no longer counts in `uptake.reads`; the event stays in `events.jsonl`; a refused relative Bash read after a move stays with no `path`, since which file it named is not known. `first_useful.steps` counts events, so a Claude Code run that prints a file through Bash before its first useful result shows one step more than before (`cat AGENTS.md`, then a passing `pytest`: 3, was 2), and `issued_s` is unchanged; `repo compare` pairs steps with batches run before this change on that basis. `docs/repo-evaluation-contract.md` §4.1, §7 and §7.1 say so. ### Added diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index c5ea0b08f..25d41cdf6 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -262,9 +262,9 @@ def command(self, words: list[str], after: str, chained: bool = False, timed: bo """Move the directory for a ``cd``; the files a printing command reads. ``chained``: the command follows ``&&`` or ``||``, so a ``cd`` in it runs only if the command before it succeeded; ``timed``: it starts with bash's ``time`` keyword (:func:`~dokimi.repos.shell_words._time_keyword`).""" - conditional = False + conditional = negated = False while words and words[0] in _KEYWORDS: # ``then cd x``: the cd may not run - words, conditional = words[1:], True + words, conditional, negated = words[1:], True, negated or words[0] == "!" words, via_env, chdir = _command_words(words, timed) if not words: return [] @@ -278,7 +278,8 @@ def command(self, words: list[str], after: str, chained: bool = False, timed: bo if after != "&": # a backgrounded cd runs in a subshell: this shell stays where it was self.here, self.shell_moved, self.moved = _cd(self.here, words[1:]), True, True elif name in _PRINTERS: - return self.printed(name, words[1:], chdir) + reads = self.printed(name, words[1:], chdir) + return [] if negated else reads # ``! cat f`` inverts the exit, which no longer says f was printed return [] def printed(self, name: str, args: list[str], chdir: bool) -> list[str]: diff --git a/dokimi/tests/test_repo_events_bash.py b/dokimi/tests/test_repo_events_bash.py index 88cf5cc1f..30af6b253 100644 --- a/dokimi/tests/test_repo_events_bash.py +++ b/dokimi/tests/test_repo_events_bash.py @@ -57,6 +57,13 @@ def test_a_failed_cat_of_one_file_is_no_read(): "output": "cat: AGENTS.md: No such file or directory", "output_truncated": False}] +@pytest.mark.parametrize("command", ["! cat AGENTS.md", "if ! cat AGENTS.md; then echo none; fi"]) +@pytest.mark.parametrize("is_error", [False, True]) +def test_a_negated_read_is_no_read(command, is_error): + # ``! cat`` of a missing file exits 0, of a present one 1: its status says nothing of the file + assert _reads(_events(_call("a", command), _result("a", "", is_error))) == [] + + @pytest.mark.parametrize("command", ["cd sub && cat AGENTS.md", "cd sub; cat AGENTS.md", "cd a && cd b && cat AGENTS.md", "cd $(git rev-parse --show-toplevel) && cat /ws/AGENTS.md", "cd /ws/sub || cat /ws/AGENTS.md", "cd sub & cat AGENTS.md", # the exit is the cat's From e4e57115321b1589695aa698421a617c530852dd Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 4 Nov 2026 12:38:44 +0000 Subject: [PATCH 42/42] fix(repo): a negated read adds nothing to unknown_reads ! cat AGENTS.md at a directory not known gave no read but counted one unknown read, as a pipe does not. Skip the printer before it is counted, and name the negated read in the contract and reviewing.md as the changelog does. Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/docs/reviewing.md | 2 +- dokimi/dokimi/repos/cli_traces.py | 5 ++--- dokimi/tests/test_repo_cli_traces.py | 4 ++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c1f4a445b..cf0c518d4 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1216,7 +1216,8 @@ per scenario). `env cd`), and exits non-zero is not a read (the exit code of `cat a missing` does not say which file failed, so both stay reads, as the known one does when the other is at a directory not known). A read - through a pipe, `grep` or `rg` is not seen, nor, under Claude Code, a relative + through a pipe, `grep` or `rg`, or negated (`! cat AGENTS.md`, whose exit + says nothing of the file), is not seen, nor, under Claude Code, a relative read after an earlier Bash call of any agent (the parent's or a subagent's, which may share its shell) ran a `cd`, `pushd`, `popd`, `eval`, `source` or `.` in its own shell (not in a subshell, pipeline or diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 745cad48e..25d48a338 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -199,7 +199,7 @@ these, in order, before writing what it showed (the fields: `cwd=`, `read=` show what loads it). `process:guidance_read` counts a read tool's reads and a Claude Code Bash call, Codex command or Copilot `bash` command that only prints files (§7's list: a bare `cat`, - `head`, `tail`, `nl` or `sed -n`), not a pipe, `grep` or `rg`, nor a + `head`, `tail`, `nl` or `sed -n`), not a pipe, `grep`, `rg` or `! cat`, nor a relative read after the shell moved; `process:unknown_reads` counts those dropped relative reads, so `guidance_read: []` with `unknown_reads` above 0 is not "never read". For the rest, look in `events.jsonl` or diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index 25d41cdf6..dc51b10a8 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -277,9 +277,8 @@ def command(self, words: list[str], after: str, chained: bool = False, timed: bo self.gated |= chained if after != "&": # a backgrounded cd runs in a subshell: this shell stays where it was self.here, self.shell_moved, self.moved = _cd(self.here, words[1:]), True, True - elif name in _PRINTERS: - reads = self.printed(name, words[1:], chdir) - return [] if negated else reads # ``! cat f`` inverts the exit, which no longer says f was printed + elif name in _PRINTERS and not negated: # ``! cat f`` inverts the exit, which no longer says f was printed + return self.printed(name, words[1:], chdir) return [] def printed(self, name: str, args: list[str], chdir: bool) -> list[str]: diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fcd6b3854..950bf9f64 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -757,3 +757,7 @@ def test_copilot_answer_falls_back_to_the_last_message_and_a_failed_exit_is_an_e t = convert("copilot-json", _jsonl({"type": "assistant.message", "data": {"content": "only"}}, {"type": "result", "exitCode": 1}), None) assert t.answer == "only" and t.is_error + + +def test_a_negated_read_at_a_directory_not_known_is_not_counted_as_unknown(): + assert shell_walk("cd -; ! cat AGENTS.md") == ([], True, 0)