feat(eval): repo-shaped workflow tasks with CI-checked reference solutions

crates/bashkit-eval/Cargo.toml

@@ -28,7 +28,7 @@ path = "src/replay_main.rs"
 # (our own agent loop + provider stack) so we do NOT need mira-everruns.
 mira-eval = { version = "0.5", features = ["macros"] }
 
-bashkit = { path = "../bashkit", features = ["jq"] }
+bashkit = { path = "../bashkit", features = ["jq", "git"] }
 bashkit-scripted-tool = { path = "../bashkit-scripted-tool", features = ["jq"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "io-std", "io-util", "fs"] }
 serde.workspace = true

crates/bashkit-eval/README.md

@@ -45,6 +45,7 @@ Results are written by mira under `./results/<run_id>/`.
 |------|---------|-----------|
 | `bashkit_bash` | 58 tasks, 15 categories | `--tag <category>`, `--samples <id>` |
 | `bashkit_smoke` | 3 tasks | quick verification |
+| `bashkit_repo` | 8 `repo_workflow` tasks | multi-turn fixture repos (`just eval-repo`) |
 | `bashkit_scripting` | scripting-tool tasks | `--axis mode=scripted\|baseline` |
 
 Targets (model matrix) are defined in `src/mira_study.rs` and gated on
@@ -58,6 +59,12 @@ not supported), e.g. `--targets anthropic/claude-opus-4-8,openai/gpt-5.5`.
 
 Smoke test dataset (`data/smoke-test.jsonl`) has 3 tasks for quick verification.
 
+Repo workflow dataset (`data/repo-workflow.jsonl`, eval `bashkit_repo`) has 8
+multi-turn tasks: a `setup` script builds a fixture git repo, the model runs
+`make test`, fixes the bug, and commits (symlinks, PATH, background jobs, jq,
+git). Reference solutions in `data/repo-workflow-solutions.jsonl` run in
+`cargo test -p bashkit-eval`, so every task stays solvable without an LLM.
+
 ## Results
 
 > Runs from 2026-06-27 onward use the mira eval framework (saved under

crates/bashkit-eval/data/repo-workflow-solutions.jsonl

@@ -0,0 +1,8 @@
+{"id": "repo_clone_fix_failing_test", "steps": ["mkdir -p /home/eval/work && cd /home/eval/work && git clone /srv/git/calc && cd calc && make test", "cd /home/eval/work/calc && cat lib/calc.sh", "cd /home/eval/work/calc && sed -i 's/^mul() { echo $(( $1 + $2 )); }/mul() { echo $(( $1 * $2 )); }/' lib/calc.sh && make test", "cd /home/eval/work/calc && git add lib/calc.sh && git commit -m 'fix: mul multiplies instead of adding' && git log"]}
+{"id": "repo_symlink_release_switch", "steps": ["cd /home/eval/deploy && cat TARGET && ls -l && make test", "cd /home/eval/deploy && ln -sfn releases/v1.5.0 current && make test", "cd /home/eval/deploy && cat current/app.conf && sed -i 's/^port = /port=/' releases/v1.5.0/app.conf && make test", "cd /home/eval/deploy && git add -A && git commit -m 'release: switch live to v1.5.0' && git log -n 1"]}
+{"id": "repo_path_bin_tool_fix", "steps": ["cd /home/eval/tools && cat README.md && make test", "cd /home/eval/tools && export PATH=\"$PWD/bin:$PATH\" && make test", "cd /home/eval/tools && sed -i 's/minor) echo \"$major.$((minor + 1)).$patch\"/minor) echo \"$major.$((minor + 1)).0\"/' bin/semver-bump && export PATH=\"$PWD/bin:$PATH\" && make test", "cd /home/eval/tools && git add bin/semver-bump && git commit -m 'fix: minor bump resets patch' && git log -n 1"]}
+{"id": "repo_parallel_shards_background", "steps": ["cd /home/eval/etl && cat SHARDS scripts/process.sh", "cd /home/eval/etl && mkdir -p out && declare -A pids && while read -r s; do bash scripts/process.sh \"$s\" & pids[$s]=$!; done < SHARDS; : > out/status.txt; while read -r s; do wait \"${pids[$s]}\"; echo \"$s $?\" >> out/status.txt; done < SHARDS; cat out/status.txt", "cd /home/eval/etl && sed -i '/^B2,n\\/a$/d' data/eu.csv && bash scripts/process.sh eu && make report", "cd /home/eval/etl && git add data out && git commit -m 'data: drop malformed eu row' && git log -n 1"]}
+{"id": "repo_report_sort_regression", "steps": ["cd /home/eval/report && make test; cat bin/top-customers", "cd /home/eval/report && sed -i 's/sort -t, -k2 -r/sort -t, -k2,2nr/' bin/top-customers && make test", "cd /home/eval/report && git commit -am 'fix: sort report totals numerically' && git log -n 1"]}
+{"id": "repo_jq_config_validation", "steps": ["cd /home/eval/svc && make test; cat build/test_configs.log; cat config/*.json", "cd /home/eval/svc && jq '.port |= tonumber' config/staging.json > /tmp/s.json && mv /tmp/s.json config/staging.json && jq '.env = \"prod\" | .features.audit = true' config/prod.json > /tmp/p.json && mv /tmp/p.json config/prod.json && make test", "cd /home/eval/svc && git add config && git commit -m 'config: fix staging port type and prod env/audit' && git log -n 1"]}
+{"id": "repo_release_branch_bump", "steps": ["cd /home/eval/lib && git checkout -b release-1.0.0 && make test; cat package.json", "cd /home/eval/lib && echo 1.0.0 > VERSION && jq '.version = \"1.0.0\"' package.json > package.json.new && mv package.json.new package.json && sed -i 's/^## Unreleased$/## 1.0.0/' CHANGELOG.md && make test", "cd /home/eval/lib && git add . && git commit -m 'release: 1.0.0' && git status"]}
+{"id": "repo_parallel_runner_masks_failure", "steps": ["cd /home/eval/parser && make test && make canary; cat tests/run.sh", "cd /home/eval/parser && cat > tests/run.sh <<'EOF'\n#!/bin/bash\n# usage: run.sh DIR -> run DIR/test_*.sh in parallel; exit 1 if any fail\ndir=$1\nmkdir -p build\npids=()\nnames=()\nfor t in \"$dir\"/test_*.sh; do\n  bash \"$t\" > \"build/$(basename \"$t\" .sh).log\" 2>&1 &\n  pids+=($!)\n  names+=(\"$t\")\ndone\nstatus=0\nfor i in \"${!pids[@]}\"; do\n  if ! wait \"${pids[$i]}\"; then echo \"FAIL ${names[$i]}\"; status=1; fi\ndone\n[ \"$status\" -eq 0 ] && echo \"all tests passed in $dir\" | tee build/last-run.txt\nexit $status\nEOF\nmake test; make canary", "cd /home/eval/parser && cat build/test_equals_in_value.log && sed -i 's/echo \"${pair##\\*=}\"/echo \"${pair#*=}\"/' src/kv.sh && cat src/kv.sh && make test && make canary", "cd /home/eval/parser && git add -A && git commit -m 'fix: runner propagates job failures; keep = in values' && git log -n 1"]}