From 672491cd4b7570cb98e448d520ed884f537bbcdb Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 13:23:38 +0000 Subject: [PATCH 1/3] fix(fs): GNU ln destinations, entries under symlinked dirs, rename keeps mode - ln: accept -n/-T/-v and long options; a directory destination (following a symlink unless -n) receives DIR/basename(TARGET); -sfn replaces a link to a directory; a real directory is never replaced (issue #1577 kept) - InMemoryFs: mkdir (non-recursive) and symlink require a directory parent, so the follow layer resolves a symlinked parent instead of creating a literal link/name entry; OverlayFs copies the parent up first - OverlayFs: file rename keeps the source mode, so sed -i on a mount_text script no longer drops its exec bit --- crates/bashkit/src/builtins/fileops.rs | 230 ++++++++++++------ crates/bashkit/src/fs/memory.rs | 15 +- crates/bashkit/src/fs/overlay.rs | 60 ++++- .../bashkit/tests/spec_cases/bash/ln.test.sh | 86 ++++++- .../tests/spec_cases/bash/symlink.test.sh | 15 ++ 5 files changed, 317 insertions(+), 89 deletions(-) diff --git a/crates/bashkit/src/builtins/fileops.rs b/crates/bashkit/src/builtins/fileops.rs index 61596bfca..bc62164f4 100644 --- a/crates/bashkit/src/builtins/fileops.rs +++ b/crates/bashkit/src/builtins/fileops.rs @@ -395,14 +395,14 @@ impl Builtin for Cp { /// Future returned by [`copy_tree`]: outer `Err` aborts, inner `Err` is a /// user-facing message. -type CopyTreeFuture<'a> = std::pin::Pin< +pub(crate) type CopyTreeFuture<'a> = std::pin::Pin< Box>> + Send + 'a>, >; /// Recursively copy `src` to `dst` (GNU `cp -R` without `-L`: symlinks are /// recreated, not followed). The outer error is cancellation/budget; the /// inner one is a user-facing message for the first failed entry. -fn copy_tree<'a>( +pub(crate) fn copy_tree<'a>( ctx: &'a Context<'_>, src: &'a Path, dst: &'a Path, @@ -976,47 +976,71 @@ impl Builtin for Chmod { /// The ln builtin - create links. /// -/// Usage: ln [-s] [-f] TARGET LINK_NAME -/// ln [-s] [-f] TARGET... DIRECTORY +/// Usage: ln [-sfnTv] TARGET LINK_NAME +/// ln [-sfnv] TARGET... DIRECTORY /// /// Options: /// -s Create symbolic link (default in Bashkit; hard links not supported in VFS) /// -f Force: remove existing destination files +/// -n Treat a symlink to a directory as the link to replace +/// -T Always treat LINK_NAME as the link itself +/// -v Print each link made /// /// Note: In Bashkit's virtual filesystem, all links are symbolic. /// Hard links are not supported; `-s` is implied. pub struct Ln; +/// Decision: GNU `ln` destination rules, symbolic links only (L-FS-001). +/// A destination that is a directory (following a symlink unless `-n`) +/// receives `DIR/basename(TARGET)`; `-n` treats a symlink to a directory as +/// the link to replace (the `ln -sfn releases/v2 current` idiom); `-T` always +/// treats the destination as the link name. A real directory is never +/// replaced, so `-f` cannot orphan its children (issue #1577). #[async_trait] impl Builtin for Ln { async fn execute(&self, ctx: Context<'_>) -> Result { if let Some(r) = super::check_help_version( ctx.args, - "Usage: ln [OPTION]... TARGET LINK_NAME\nCreate a link to TARGET with the name LINK_NAME.\n\n -s\t\tmake symbolic links instead of hard links\n -f\t\tremove existing destination files\n --help\tdisplay this help and exit\n --version\toutput version information and exit\n", + "Usage: ln [OPTION]... TARGET LINK_NAME\n or: ln [OPTION]... TARGET... DIRECTORY\nCreate a link to TARGET with the name LINK_NAME, or links in DIRECTORY.\n\n -s, --symbolic\t\tmake symbolic links instead of hard links\n -f, --force\t\t\tremove existing destination files\n -n, --no-dereference\t\ttreat LINK_NAME as a normal file if it is a\n\t\t\t\tsymbolic link to a directory\n -T, --no-target-directory\ttreat LINK_NAME as a normal file always\n -v, --verbose\t\t\tprint name of each linked file\n --help\tdisplay this help and exit\n --version\toutput version information and exit\n", Some("ln (bashkit) 0.1"), ) { return Ok(r); } let mut force = false; + let mut no_deref = false; + let mut no_target_dir = false; + let mut verbose = false; let mut files: Vec<&str> = Vec::new(); + let mut opts_done = false; for arg in ctx.args.iter() { - if arg.starts_with('-') && arg.len() > 1 { + if opts_done || arg == "-" || !arg.starts_with('-') { + files.push(arg); + } else if arg == "--" { + opts_done = true; + } else if let Some(long) = arg.strip_prefix("--") { + match long { + "symbolic" | "logical" | "physical" => {} + "force" => force = true, + "no-dereference" => no_deref = true, + "no-target-directory" => no_target_dir = true, + "verbose" => verbose = true, + _ => return Ok(super::invalid_option("ln", arg, 1)), + } + } else { for c in arg[1..].chars() { match c { - 's' => {} // symbolic — always symbolic in VFS + // symbolic: always symbolic in the VFS; -L/-P only + // matter for hard links. + 's' | 'L' | 'P' => {} 'f' => force = true, - _ => { - return Ok(ExecResult::err( - format!("ln: invalid option -- '{}'\n", c), - 1, - )); - } + 'n' => no_deref = true, + 'T' => no_target_dir = true, + 'v' => verbose = true, + _ => return Ok(super::invalid_option("ln", &format!("-{c}"), 1)), } } - } else { - files.push(arg); } } @@ -1024,46 +1048,92 @@ impl Builtin for Ln { return Ok(ExecResult::err("ln: missing file operand\n".to_string(), 1)); } - let target = files[0]; - let link_name = files[1]; - let link_path = resolve_path(ctx.cwd, link_name); - - // If link already exists - if ctx.fs.exists(&link_path).await.unwrap_or(false) { - if force { - // Surface remove failures (e.g. non-empty directory) instead - // of falling through and overwriting via symlink, which on - // the in-memory VFS would orphan children and corrupt state - // (issue #1577). - if let Err(e) = ctx.fs.remove(&link_path, false).await { - return Ok(ExecResult::err( - format!("ln: cannot remove '{}': {}\n", link_name, e), - 1, - )); - } - } else { + let (targets, dest) = files.split_at(files.len() - 1); + let dest = dest[0]; + let dest_path = resolve_path(ctx.cwd, dest); + + let into_dir = if no_target_dir { + if targets.len() > 1 { return Ok(ExecResult::err( - format!( - "ln: failed to create symbolic link '{}': File exists\n", - link_name - ), + format!("ln: extra operand '{}'\n", dest), 1, )); } + false + } else { + let meta = if no_deref { + ctx.fs.lstat(&dest_path).await + } else { + ctx.fs.stat(&dest_path).await + }; + matches!(meta, Ok(m) if m.file_type.is_dir()) + }; + if targets.len() > 1 && !into_dir { + return Ok(ExecResult::err( + format!("ln: target '{}' is not a directory\n", dest), + 1, + )); } - let target_path = Path::new(target); - if let Err(e) = ctx.fs.symlink(target_path, &link_path).await { - return Ok(ExecResult::err( - format!( + let mut out = String::new(); + let mut errors = String::new(); + for target in targets { + let (link_name, link_path) = if into_dir { + let base = target + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or(target); + let shown = if dest.ends_with('/') { + format!("{dest}{base}") + } else { + format!("{dest}/{base}") + }; + (shown, vfs_join(&dest_path, base)) + } else { + (dest.to_string(), dest_path.clone()) + }; + + if let Ok(meta) = ctx.fs.lstat(&link_path).await { + if !force { + errors.push_str(&format!( + "ln: failed to create symbolic link '{}': File exists\n", + link_name + )); + continue; + } + if meta.file_type.is_dir() { + errors.push_str(&format!( + "ln: '{}': cannot overwrite directory\n", + link_name + )); + continue; + } + if let Err(e) = ctx.fs.remove(&link_path, false).await { + errors.push_str(&format!("ln: cannot remove '{}': {}\n", link_name, e)); + continue; + } + } + + if let Err(e) = ctx.fs.symlink(Path::new(target), &link_path).await { + errors.push_str(&format!( "ln: failed to create symbolic link '{}': {}\n", link_name, e - ), - 1, - )); + )); + continue; + } + if verbose { + out.push_str(&format!("'{}' -> '{}'\n", link_name, target)); + } } - Ok(ExecResult::ok(String::new())) + if errors.is_empty() { + Ok(ExecResult::ok(out)) + } else { + let mut r = ExecResult::err(errors, 1); + r.stdout = out.into(); + Ok(r) + } } } @@ -1668,12 +1738,11 @@ mod tests { assert_eq!(meta.mode, 0o755); } - /// Regression: issue #1577. `ln -f` over a non-empty directory must - /// surface the underlying remove failure instead of silently - /// proceeding to symlink creation, which on the in-memory VFS would - /// orphan the directory's children. + /// Regression: issue #1577. `ln -f` must never replace a non-empty + /// directory (on the in-memory VFS that would orphan its children). Like + /// GNU ln, the link goes inside the directory; with `-T` ln refuses. #[tokio::test] - async fn test_ln_force_over_non_empty_dir_fails() { + async fn test_ln_force_over_non_empty_dir_keeps_children() { let (fs, mut cwd, mut variables) = create_test_ctx().await; let env = HashMap::new(); @@ -1686,37 +1755,36 @@ mod tests { .await .unwrap(); - let args = vec![ - "-sf".to_string(), - "target.txt".to_string(), - "destdir".to_string(), - ]; - let ctx = Context { - args: &args, - env: &env, - variables: &mut variables, - cwd: &mut cwd, - fs: fs.clone(), - stdin: None, - #[cfg(feature = "http_client")] - http_client: None, - #[cfg(feature = "git")] - git_client: None, - #[cfg(feature = "ssh")] - ssh_client: None, - shell: None, - }; - - let result = Ln.execute(ctx).await.unwrap(); - assert_ne!( - result.exit_code, 0, - "ln -sf must fail when destination is a non-empty directory" - ); - // Child must still exist; ln must not have proceeded to symlink. - assert!( - fs.exists(&cwd.join("destdir/child.txt")).await.unwrap(), - "child file must not be orphaned" - ); + for (flags, want_exit) in [("-sf", 0), ("-sfT", 1)] { + let args = vec![ + flags.to_string(), + "target.txt".to_string(), + "destdir".to_string(), + ]; + let ctx = Context { + args: &args, + env: &env, + variables: &mut variables, + cwd: &mut cwd, + fs: fs.clone(), + stdin: None, + #[cfg(feature = "http_client")] + http_client: None, + #[cfg(feature = "git")] + git_client: None, + #[cfg(feature = "ssh")] + ssh_client: None, + shell: None, + }; + let result = Ln.execute(ctx).await.unwrap(); + assert_eq!(result.exit_code, want_exit, "ln {flags}"); + assert!( + fs.exists(&cwd.join("destdir/child.txt")).await.unwrap(), + "child file must not be orphaned (ln {flags})" + ); + } + let link = fs.read_link(&cwd.join("destdir/target.txt")).await.unwrap(); + assert_eq!(link, std::path::PathBuf::from("target.txt")); } #[test] diff --git a/crates/bashkit/src/fs/memory.rs b/crates/bashkit/src/fs/memory.rs index 4a563e9b9..48377eaba 100644 --- a/crates/bashkit/src/fs/memory.rs +++ b/crates/bashkit/src/fs/memory.rs @@ -1492,10 +1492,12 @@ impl FileSystem for InMemoryFs { } } } else { - // Check parent exists + // Parent must be a directory. A symlink parent is refused too, so + // the symlink-following layer resolves it and the entry lands in + // the link's target instead of under a literal `link/` key. if let Some(parent) = path.parent() - && !entries.contains_key(parent) && parent != Path::new("/") + && !matches!(entries.get(parent), Some(FsEntry::Directory { .. })) { return Err(IoError::new(ErrorKind::NotFound, "parent directory not found").into()); } @@ -1781,6 +1783,15 @@ impl FileSystem for InMemoryFs { let target_size = target.as_os_str().as_encoded_bytes().len(); let mut entries = self.entries.write().unwrap(); + // Parent must be a directory (see `mkdir`): a symlink parent is left to + // the following layer to resolve. + if let Some(parent) = link.parent() + && parent != Path::new("/") + && !matches!(entries.get(parent), Some(FsEntry::Directory { .. })) + { + return Err(IoError::new(ErrorKind::NotFound, "parent directory not found").into()); + } + // THREAT[TM-DOS-045]: Symlinks count toward file count. // THREAT[TM-DOS-013]: Symlink target bytes count toward filesystem usage. // check_write_limits enforces single-file size, total bytes, and file count. diff --git a/crates/bashkit/src/fs/overlay.rs b/crates/bashkit/src/fs/overlay.rs index 91422e0c7..16affd6c0 100644 --- a/crates/bashkit/src/fs/overlay.rs +++ b/crates/bashkit/src/fs/overlay.rs @@ -183,6 +183,22 @@ pub struct OverlayFs { } impl OverlayFs { + /// Make sure `path`'s parent exists in the upper layer, copying the + /// directory chain up from the lower layer like `write_file` does. The + /// upper `InMemoryFs` refuses entries whose parent it does not hold. + async fn ensure_upper_parent(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() + && !self.upper.exists(parent).await.unwrap_or(false) + { + if self.lower.exists(parent).await.unwrap_or(false) { + self.upper.mkdir(parent, true).await?; + } else { + return Err(IoError::new(ErrorKind::NotFound, "parent directory not found").into()); + } + } + Ok(()) + } + /// Create a new overlay filesystem with the given base layer and default limits. /// /// The `lower` filesystem is treated as read-only - all reads will first @@ -912,14 +928,17 @@ impl FileSystem for OverlayFs { let target = self.read_link(&from).await?; self.check_write_limits(0)?; self.remove_whiteout(&to); + self.ensure_upper_parent(&to).await?; self.upper.symlink(&target, &to).await?; self.remove(&from, false).await?; return Ok(()); } - // Regular file: read content and write to new location + // Regular file: read content and write to new location. A rename + // keeps the inode's mode (`sed -i` relies on it for scripts). let content = self.read_file(&from).await?; self.write_file(&to, &content).await?; + self.chmod(&to, meta.mode).await?; self.remove(&from, false).await?; Ok(()) @@ -942,6 +961,7 @@ impl FileSystem for OverlayFs { let target = self.read_link(&from).await?; self.check_write_limits(0)?; self.remove_whiteout(&to); + self.ensure_upper_parent(&to).await?; return self.upper.symlink(&target, &to).await; } @@ -965,6 +985,7 @@ impl FileSystem for OverlayFs { self.remove_whiteout(&link); // Create symlink in upper + self.ensure_upper_parent(&link).await?; self.upper.symlink(target, &link).await } @@ -1248,6 +1269,43 @@ mod tests { assert!(!lower.exists(Path::new("/tmp/new.txt")).await.unwrap()); } + /// `sed -i` replaces a file by renaming a chmod-ed temp over it; a plain + /// file rename must keep the source mode or an executable script loses + /// its exec bit (found by the eval repo_workflow reference solutions). + #[tokio::test] + async fn test_rename_file_preserves_mode() { + let lower = Arc::new(InMemoryFs::new()); + lower + .write_file(Path::new("/tmp/lower.sh"), b"#!/bin/bash\n") + .await + .unwrap(); + lower + .chmod(Path::new("/tmp/lower.sh"), 0o755) + .await + .unwrap(); + let overlay = OverlayFs::new(lower); + overlay + .write_file(Path::new("/tmp/upper.sh"), b"#!/bin/bash\n") + .await + .unwrap(); + overlay + .chmod(Path::new("/tmp/upper.sh"), 0o750) + .await + .unwrap(); + + for (from, to, mode) in [ + ("/tmp/lower.sh", "/tmp/a.sh", 0o755), + ("/tmp/upper.sh", "/tmp/b.sh", 0o750), + ] { + overlay + .rename(Path::new(from), Path::new(to)) + .await + .unwrap(); + let meta = overlay.stat(Path::new(to)).await.unwrap(); + assert_eq!(meta.mode & 0o777, mode, "{from} -> {to}"); + } + } + #[tokio::test] async fn test_copy_on_write() { let lower = Arc::new(InMemoryFs::new()); diff --git a/crates/bashkit/tests/spec_cases/bash/ln.test.sh b/crates/bashkit/tests/spec_cases/bash/ln.test.sh index cb1920791..bab836458 100644 --- a/crates/bashkit/tests/spec_cases/bash/ln.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/ln.test.sh @@ -47,14 +47,90 @@ echo "ok" ok ### end -### ln_force_dir_dest_fails -### exit_code:1 -### bash_diff: Bashkit VFS rejects ln -sf over a non-empty directory; real ln also fails here. -# Regression: issue #1577. ln -f must not silently overwrite a non-empty -# directory with a symlink — that would orphan its children in the VFS. +### ln_force_dir_dest_links_inside +# Regression: issue #1577. ln -f must never replace a non-empty directory +# with a symlink (that would orphan its children in the VFS). Like GNU ln, a +# directory destination receives the link inside it instead. echo content > /tmp/force_dir_target.txt mkdir -p /tmp/force_dir_dest echo child > /tmp/force_dir_dest/child.txt ln -sf /tmp/force_dir_target.txt /tmp/force_dir_dest +cat /tmp/force_dir_dest/child.txt +readlink /tmp/force_dir_dest/force_dir_target.txt +### expect +child +/tmp/force_dir_target.txt +### end + +### ln_force_T_over_directory_fails +### exit_code:1 +# -T treats the destination as the link itself; a real directory is never replaced +mkdir -p /tmp/lnTdir/dest +ln -sfT /tmp/lnTdir/x /tmp/lnTdir/dest 2>/dev/null +### expect +### end + +### ln_sfn_replaces_dir_symlink +# ln -sfn swaps a symlink that points at a directory (release switch idiom) +mkdir -p /tmp/lnn/releases/v1 /tmp/lnn/releases/v2 +cd /tmp/lnn +ln -s releases/v1 current +ln -sfn releases/v2 current +readlink current +ls releases/v1 +### expect +releases/v2 +### end + +### ln_sf_follows_dir_symlink +# without -n an existing link to a directory is followed: the new link lands inside it +mkdir -p /tmp/lnd/releases/v1 /tmp/lnd/releases/v2 +cd /tmp/lnd +ln -s releases/v1 current +ln -sf releases/v2 current +readlink current +ls releases/v1 +### expect +releases/v1 +v2 +### end + +### ln_into_directory +# a directory destination gets DIR/basename(TARGET) +mkdir -p /tmp/lni/d +echo x > /tmp/lni/t.txt +ln -s /tmp/lni/t.txt /tmp/lni/d +readlink /tmp/lni/d/t.txt +### expect +/tmp/lni/t.txt +### end + +### ln_multiple_targets_into_directory +mkdir -p /tmp/lnm/d +cd /tmp/lnm +ln -s /x/a /x/b d +readlink d/a d/b +### expect +/x/a +/x/b +### end + +### ln_sfT_no_target_directory +mkdir -p /tmp/lnt/a /tmp/lnt/b +cd /tmp/lnt +ln -s a cur +ln -sfT b cur +readlink cur +### expect +b +### end + +### ln_verbose_and_long_options +cd /tmp +ln --symbolic --verbose tgt_v lnk_v +ln --symbolic --force --no-dereference tgt_w lnk_v +readlink lnk_v ### expect +'lnk_v' -> 'tgt_v' +tgt_w ### end diff --git a/crates/bashkit/tests/spec_cases/bash/symlink.test.sh b/crates/bashkit/tests/spec_cases/bash/symlink.test.sh index dc180907f..8e2281b0c 100644 --- a/crates/bashkit/tests/spec_cases/bash/symlink.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/symlink.test.sh @@ -131,3 +131,18 @@ readlink -f l2 deep /tmp/sl9/a/b/f ### end + +### symlink_parent_dir_mkdir_and_ln_land_in_target +# mkdir and ln -s under a symlinked directory create entries in the link's target +mkdir -p /tmp/slp/real +cd /tmp/slp +ln -s real via +mkdir via/sub +ln -s ../elsewhere via/lnk +ls real +readlink real/lnk +### expect +lnk +sub +../elsewhere +### end From 507cf83803803a42a7b744f9d306de9eb04773db Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 13:23:38 +0000 Subject: [PATCH 2/3] fix(git): add -A, commit -am, and clone from a local VFS path - add -A/--all/-u stage the whole tree (they were taken as paths and staged nothing); other flags are ignored instead of treated as paths - commit parses -a/--all and short bundles like -am MSG, restaging every tracked file before committing - clone PATH|file://PATH [DIR] copies an in-VFS repo with the budgeted cp -R walker and sets origin to the source; refuses missing repos, non-empty destinations and destinations inside the source --- crates/bashkit/src/builtins/git/cmd.rs | 186 +++++++++++++++++- .../integration/git_integration_tests.rs | 119 +++++++++++ knowledge/integrations/git-support.md | 16 +- knowledge/security/threat-model.md | 2 +- 4 files changed, 311 insertions(+), 12 deletions(-) diff --git a/crates/bashkit/src/builtins/git/cmd.rs b/crates/bashkit/src/builtins/git/cmd.rs index d59dbaa90..441a33864 100644 --- a/crates/bashkit/src/builtins/git/cmd.rs +++ b/crates/bashkit/src/builtins/git/cmd.rs @@ -290,7 +290,26 @@ async fn git_add( )); } - let paths: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); + // `-A`/`--all`/`-u`/`--update` stage the whole tree (there is no content + // tracking, so "modified tracked files" is every file); other flags + // (`-v`, `-f`, ...) are accepted and ignored. `--` ends options. + let mut paths: Vec<&str> = Vec::new(); + let mut opts_done = false; + for arg in args { + if opts_done || !arg.starts_with('-') || arg == "-" { + paths.push(arg); + } else if arg == "--" { + opts_done = true; + } else if matches!(arg.as_str(), "-A" | "--all" | "-u" | "--update") { + paths.push("."); + } + } + if paths.is_empty() { + return Ok(ExecResult::err( + "Nothing specified, nothing added.\n".to_string(), + 0, + )); + } match git_client.add(&ctx.fs, ctx.cwd, &paths).await { Ok(()) => Ok(ExecResult::ok(String::new())), @@ -304,23 +323,59 @@ async fn git_commit( git_client: &super::GitClient, args: &[String], ) -> Result { - // Parse -m + // Parse -m and -a, including bundles like `-am `. let mut message: Option = None; + let mut all = false; let mut i = 0; while i < args.len() { let arg = &args[i]; - if arg == "-m" { + if arg == "--all" { + all = true; + } else if let Some(msg) = arg.strip_prefix("--message=") { + message = Some(msg.to_string()); + } else if arg == "--message" { if i + 1 < args.len() { message = Some(args[i + 1].clone()); i += 1; } - } else if let Some(msg) = arg.strip_prefix("-m") { - message = Some(msg.to_string()); + } else if arg.starts_with('-') && !arg.starts_with("--") && arg.len() > 1 { + // Short-flag bundle: letters until `m`, whose value is the rest + // of the bundle or the next argument. + for (pos, c) in arg[1..].char_indices() { + match c { + 'a' => all = true, + 'm' => { + let rest = &arg[1 + pos + 1..]; + if !rest.is_empty() { + message = Some(rest.to_string()); + } else if i + 1 < args.len() { + message = Some(args[i + 1].clone()); + i += 1; + } + break; + } + _ => {} + } + } } i += 1; } + // `-a`: stage every tracked file still present before committing. + if all { + let tracked = match git_client.ls_files(&ctx.fs, ctx.cwd).await { + Ok(files) => files, + Err(e) => return git_err(e), + }; + let paths: Vec<&str> = tracked.iter().map(String::as_str).collect(); + if !paths.is_empty() + && let Err(e) = git_client.add(&ctx.fs, ctx.cwd, &paths).await + { + return git_err(e); + } + } + let message = match message { Some(m) => m, None => match editor_message(&ctx).await? { @@ -540,16 +595,33 @@ async fn git_clone( git_client: &super::GitClient, args: &[String], ) -> Result { - if args.is_empty() { + // Positional operands; flags are accepted and ignored (`--depth N`, + // `-b BRANCH`, `--origin NAME` consume their value). + let mut operands: Vec<&String> = Vec::new(); + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + match arg.as_str() { + "--depth" | "-b" | "--branch" | "-o" | "--origin" => i += 1, + a if a.starts_with('-') && a.len() > 1 => {} + _ => operands.push(arg), + } + i += 1; + } + if operands.is_empty() { return Ok(ExecResult::err( "usage: git clone []\n".to_string(), 129, )); } - let url = &args[0]; - let dest = if args.len() > 1 { - resolve_path(ctx.cwd, &args[1]) + let url = operands[0]; + if let Some(src) = local_clone_source(url) { + let quiet = args.iter().any(|a| a == "-q" || a == "--quiet"); + return git_clone_local(&ctx, src, operands.get(1).map(|s| s.as_str()), quiet).await; + } + let dest = if operands.len() > 1 { + resolve_path(ctx.cwd, operands[1]) } else { // Extract repo name from URL let name = url @@ -566,6 +638,102 @@ async fn git_clone( } } +/// A clone source that names a repository in the VFS: `file://PATH` or a +/// plain path (anything without a URL scheme or an scp-style `host:`). +#[cfg(feature = "git")] +fn local_clone_source(url: &str) -> Option<&str> { + if let Some(path) = url.strip_prefix("file://") { + return Some(path); + } + if url.contains("://") { + return None; + } + let before_slash = url.split('/').next().unwrap_or(url); + (!before_slash.contains(':')).then_some(url) +} + +/// `git clone [dir]`: copy a working tree plus its `.git` state. +/// +/// Decision: the repository already lives in the VFS, so a local clone is a +/// tree copy (symlinks recreated, budget-accounted via `copy_tree`) with +/// `origin` pointed at the source path. No network, no host path: the source +/// resolves in the VFS like any other path (TM-GIT-005), and FS limits bound +/// the copy (TM-GIT-007/009). Only non-bare repos exist in this VFS format. +#[cfg(feature = "git")] +async fn git_clone_local( + ctx: &Context<'_>, + src_arg: &str, + dest_arg: Option<&str>, + quiet: bool, +) -> Result { + let src = resolve_path(ctx.cwd, src_arg); + let src_git = crate::fs::vfs_join(&src, ".git"); + if !matches!(ctx.fs.stat(&src_git).await, Ok(m) if m.file_type.is_dir()) { + return Ok(ExecResult::err( + format!("fatal: repository '{src_arg}' does not exist\n"), + 128, + )); + } + + let default_name; + let dest_arg = match dest_arg { + Some(d) => d, + None => { + default_name = src_arg + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or("repo") + .trim_end_matches(".git") + .to_string(); + default_name.as_str() + } + }; + let dest = resolve_path(ctx.cwd, dest_arg); + if dest.starts_with(&src) { + return Ok(ExecResult::err( + format!("fatal: cannot clone '{src_arg}' into itself ('{dest_arg}')\n"), + 128, + )); + } + if let Ok(meta) = ctx.fs.stat(&dest).await { + let empty_dir = meta.file_type.is_dir() + && ctx + .fs + .read_dir(&dest) + .await + .map(|e| e.is_empty()) + .unwrap_or(false); + if !empty_dir { + return Ok(ExecResult::err( + format!( + "fatal: destination path '{dest_arg}' already exists and is not an empty directory.\n" + ), + 128, + )); + } + } + if let Some(parent) = dest.parent() + && !ctx.fs.exists(parent).await? + { + ctx.fs.mkdir(parent, true).await?; + } + + if let Err(msg) = crate::builtins::fileops::copy_tree(ctx, &src, &dest, false).await? { + return Ok(ExecResult::err(format!("fatal: {msg}\n"), 128)); + } + let remotes = crate::fs::vfs_join(&dest, ".git/remotes"); + ctx.fs + .write_file(&remotes, format!("origin|{}\n", src.display()).as_bytes()) + .await?; + + let mut result = ExecResult::ok(String::new()); + if !quiet { + result.stderr = format!("Cloning into '{dest_arg}'...\ndone.\n").into(); + } + Ok(result) +} + #[cfg(feature = "git")] async fn git_fetch( ctx: Context<'_>, diff --git a/crates/bashkit/tests/integration/git_integration_tests.rs b/crates/bashkit/tests/integration/git_integration_tests.rs index 6f57827cf..14e1a8953 100644 --- a/crates/bashkit/tests/integration/git_integration_tests.rs +++ b/crates/bashkit/tests/integration/git_integration_tests.rs @@ -405,3 +405,122 @@ async fn git_init_options_are_parsed_not_taken_as_directory() { let r = bash.exec("git init -b 'a b' /y").await.unwrap(); assert_eq!(r.exit_code, 128); } + +/// Stage-all shorthands agents reach for first (`add -A`, `commit -am`). +mod stage_all { + use super::*; + + #[tokio::test] + async fn test_git_add_all_flags_stage_everything() { + for flag in ["-A", "--all"] { + let mut bash = create_git_bash(); + let script = format!( + "mkdir -p /r/d && cd /r && git init -q && echo a > f && echo b > d/g && \ + git add {flag} && git commit -m init && git ls-files" + ); + let result = bash.exec(&script).await.unwrap(); + assert_eq!(result.exit_code, 0, "{flag}: {}", result.stderr); + assert!(result.stdout.contains("d/g"), "{flag}: {}", result.stdout); + assert!(result.stdout.contains("f\n"), "{flag}: {}", result.stdout); + } + } + + #[tokio::test] + async fn test_git_commit_am_stages_tracked_files_only() { + let mut bash = create_git_bash(); + let result = bash + .exec( + "mkdir -p /r && cd /r && git init -q && echo a > f && git add . && \ + git commit -q -m init && echo b >> f && echo n > untracked && \ + git commit -am 'fix: y'", + ) + .await + .unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert!(result.stdout.contains("fix: y"), "{}", result.stdout); + + let log = bash.exec("cd /r && git log").await.unwrap(); + assert!(log.stdout.contains("fix: y"), "{}", log.stdout); + let files = bash.exec("cd /r && git ls-files").await.unwrap(); + assert!(!files.stdout.contains("untracked"), "{}", files.stdout); + + // Split flags work the same way. + let split = bash + .exec("cd /r && echo c >> f && git commit -a -m 'second'") + .await + .unwrap(); + assert_eq!(split.exit_code, 0, "{}", split.stderr); + } +} + +/// `git clone `: copy a repo that already lives in the VFS. +mod clone_local { + use super::*; + + const UPSTREAM: &str = "mkdir -p /srv/git/calc/lib && cd /srv/git/calc && git init -q && \ + echo 'mul() { :; }' > lib/calc.sh && ln -s lib/calc.sh entry && \ + git add . && git commit -q -m 'initial import' && cd /"; + + #[tokio::test] + async fn test_git_clone_local_path_copies_tree_and_history() { + let mut bash = create_git_bash(); + bash.exec(UPSTREAM).await.unwrap(); + let r = bash + .exec("mkdir -p /work && cd /work && git clone /srv/git/calc") + .await + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stderr.contains("Cloning into 'calc'"), "{}", r.stderr); + + let r = bash + .exec("cd /work/calc && cat lib/calc.sh && readlink entry && git log && git remote -v") + .await + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.contains("mul()"), "{}", r.stdout); + assert!(r.stdout.contains("lib/calc.sh\n"), "{}", r.stdout); + assert!(r.stdout.contains("initial import"), "{}", r.stdout); + assert!(r.stdout.contains("origin\t/srv/git/calc"), "{}", r.stdout); + + // The clone is independent of upstream. + bash.exec("cd /work/calc && echo changed > lib/calc.sh") + .await + .unwrap(); + let up = bash.exec("cat /srv/git/calc/lib/calc.sh").await.unwrap(); + assert!(up.stdout.contains("mul()"), "{}", up.stdout); + } + + #[tokio::test] + async fn test_git_clone_local_explicit_dest_and_file_url() { + let mut bash = create_git_bash(); + bash.exec(UPSTREAM).await.unwrap(); + let r = bash + .exec("git clone -q file:///srv/git/calc /w/c2 && cat /w/c2/lib/calc.sh") + .await + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.contains("mul()"), "{}", r.stdout); + } + + #[tokio::test] + async fn test_git_clone_local_errors() { + let mut bash = create_git_bash(); + bash.exec(UPSTREAM).await.unwrap(); + let r = bash.exec("git clone /srv/git/nope /w/x").await.unwrap(); + assert_eq!(r.exit_code, 128); + assert!(r.stderr.contains("does not exist"), "{}", r.stderr); + + let r = bash + .exec("mkdir -p /w/full && touch /w/full/f && git clone /srv/git/calc /w/full") + .await + .unwrap(); + assert_eq!(r.exit_code, 128); + assert!(r.stderr.contains("already exists"), "{}", r.stderr); + + let r = bash + .exec("git clone /srv/git/calc /srv/git/calc/inner") + .await + .unwrap(); + assert_eq!(r.exit_code, 128, "{}", r.stderr); + } +} diff --git a/knowledge/integrations/git-support.md b/knowledge/integrations/git-support.md index c6e3647b1..15d7f131c 100644 --- a/knowledge/integrations/git-support.md +++ b/knowledge/integrations/git-support.md @@ -21,14 +21,26 @@ on the virtual filesystem only. Configure via ### Supported Commands -- Phase 1 (local): `init`, `config`, `add`, `commit -m` (no `-m` opens `$EDITOR` inside a - terminal session, see [In-Process Terminal](in-process-terminal.md)), `status`, `log [-n N]` +- Phase 1 (local): `init`, `config`, `add` (`-A`/`--all`/`-u` stage the whole + tree), `commit -m` (`-a`/`-am MSG` restage every tracked file first; no `-m` + opens `$EDITOR` inside a terminal session, see + [In-Process Terminal](in-process-terminal.md)), `status`, `log [-n N]` +- Local clone: `git clone PATH|file://PATH [DIR]` copies a repo that already + lives in the VFS (tree + `.git`, symlinks kept, via `cp -R`'s budgeted + `copy_tree`) and points `origin` at the source path. Refuses a missing repo, + a non-empty destination, or a destination inside the source (exit 128). - Phase 2 (remote, virtual mode): `remote [-v]`, `remote add/remove` (fully functional); `clone`/`push`/`pull`/`fetch` validate URL against allowlist then return virtual-mode messages (no network in VFS-only mode) - Phase 3 (advanced): `branch [-d]`, `checkout [-b]`, `diff` (simplified), `reset [--soft|--mixed|--hard]` - Future: `merge`, `rebase`, `stash` +- Known gaps (seen by the `repo_workflow` eval): no content tracking, so + `status` never shows modified files and `commit -a` always commits; commit + hashes derive from the timestamp (two commits in one second share a hash); + `commit` always advances `refs/heads/master` and prints `[master ...]`, even + on another branch; `log --oneline`/`--format` and `status --porcelain` are + ignored (full format); `commit -q` still prints; `show --stat` fails. ### Security diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 082fdae97..a70927beb 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -1040,7 +1040,7 @@ security threats related to git operations and their mitigations. | TM-GIT-002 | Host identity leak | Commit reveals real name/email | Configurable virtual identity | **MITIGATED** | | TM-GIT-003 | Host git config access | Read ~/.gitconfig | No host filesystem access | **MITIGATED** | | TM-GIT-004 | Credential theft | Access git credential store | No host filesystem access | **MITIGATED** | -| TM-GIT-005 | Repository escape | `git clone` outside VFS | All paths in VFS | **MITIGATED** | +| TM-GIT-005 | Repository escape | `git clone` outside VFS | All paths in VFS; a local clone source (`PATH`, `file://PATH`) resolves in the VFS and is copied with the budgeted `cp -R` walker, bounded by FS limits | **MITIGATED** | **Current Risk**: LOW. Implementation: `git/client.rs`, author identity (`[user]` name/email) is built from configurable `self.config` values, never read from host `~/.gitconfig` (TM-GIT-002). From 9adbce0cdf22e80b303ca5e712d1337c919e2658 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 13:23:39 +0000 Subject: [PATCH 3/3] feat(eval): repo_workflow tasks with CI-checked reference solutions New bashkit_repo eval (just eval-repo): 8 multi-turn repo tasks where a setup script builds a fixture git repo and the model runs make test, fixes the bug, and commits. Tasks combine local clone, symlinks (ln -sfn), PATH tweaks, background jobs with per-PID wait, jq/awk/sed, and git. - EvalTask gains optional setup (subshell, set -e, unscored); the eval Bash enables sandboxed git; agent and tests share build_task_bash - new checks: file_not_contains, symlink (relative or absolute target) - reference.rs replays one reference solution per task through the real checks; cargo test requires each to pass and the untouched fixture to fail --- crates/bashkit-eval/Cargo.toml | 2 +- crates/bashkit-eval/README.md | 7 + .../data/repo-workflow-solutions.jsonl | 8 + crates/bashkit-eval/data/repo-workflow.jsonl | 8 + crates/bashkit-eval/src/agent.rs | 48 +++++- crates/bashkit-eval/src/checks.rs | 121 ++++++++++++++ crates/bashkit-eval/src/dataset.rs | 5 + crates/bashkit-eval/src/lib.rs | 1 + crates/bashkit-eval/src/main.rs | 5 + crates/bashkit-eval/src/mira_study.rs | 39 ++++- crates/bashkit-eval/src/reference.rs | 155 ++++++++++++++++++ crates/bashkit-eval/src/snapshot.rs | 34 +++- justfile | 5 + knowledge/log.md | 2 + knowledge/operations/eval.md | 59 ++++++- 15 files changed, 484 insertions(+), 15 deletions(-) create mode 100644 crates/bashkit-eval/data/repo-workflow-solutions.jsonl create mode 100644 crates/bashkit-eval/data/repo-workflow.jsonl create mode 100644 crates/bashkit-eval/src/reference.rs diff --git a/crates/bashkit-eval/Cargo.toml b/crates/bashkit-eval/Cargo.toml index 10c39cf27..f0bed0634 100644 --- a/crates/bashkit-eval/Cargo.toml +++ b/crates/bashkit-eval/Cargo.toml @@ -28,7 +28,7 @@ path = "src/replay_main.rs" # (our own agent loop + provider stack) so we do NOT need mira-everruns. mira-eval = { version = "0.5", features = ["macros"] } -bashkit = { path = "../bashkit", features = ["jq"] } +bashkit = { path = "../bashkit", features = ["jq", "git"] } bashkit-scripted-tool = { path = "../bashkit-scripted-tool", features = ["jq"] } tokio = { workspace = true, features = ["rt-multi-thread", "io-std", "io-util", "fs"] } serde.workspace = true diff --git a/crates/bashkit-eval/README.md b/crates/bashkit-eval/README.md index 2a67a06ce..82725e600 100644 --- a/crates/bashkit-eval/README.md +++ b/crates/bashkit-eval/README.md @@ -45,6 +45,7 @@ Results are written by mira under `./results//`. |------|---------|-----------| | `bashkit_bash` | 58 tasks, 15 categories | `--tag `, `--samples ` | | `bashkit_smoke` | 3 tasks | quick verification | +| `bashkit_repo` | 8 `repo_workflow` tasks | multi-turn fixture repos (`just eval-repo`) | | `bashkit_scripting` | scripting-tool tasks | `--axis mode=scripted\|baseline` | Targets (model matrix) are defined in `src/mira_study.rs` and gated on @@ -58,6 +59,12 @@ not supported), e.g. `--targets anthropic/claude-opus-4-8,openai/gpt-5.5`. Smoke test dataset (`data/smoke-test.jsonl`) has 3 tasks for quick verification. +Repo workflow dataset (`data/repo-workflow.jsonl`, eval `bashkit_repo`) has 8 +multi-turn tasks: a `setup` script builds a fixture git repo, the model runs +`make test`, fixes the bug, and commits (symlinks, PATH, background jobs, jq, +git). Reference solutions in `data/repo-workflow-solutions.jsonl` run in +`cargo test -p bashkit-eval`, so every task stays solvable without an LLM. + ## Results > Runs from 2026-06-27 onward use the mira eval framework (saved under diff --git a/crates/bashkit-eval/data/repo-workflow-solutions.jsonl b/crates/bashkit-eval/data/repo-workflow-solutions.jsonl new file mode 100644 index 000000000..4ed649a63 --- /dev/null +++ b/crates/bashkit-eval/data/repo-workflow-solutions.jsonl @@ -0,0 +1,8 @@ +{"id": "repo_clone_fix_failing_test", "steps": ["mkdir -p /home/eval/work && cd /home/eval/work && git clone /srv/git/calc && cd calc && make test", "cd /home/eval/work/calc && cat lib/calc.sh", "cd /home/eval/work/calc && sed -i 's/^mul() { echo $(( $1 + $2 )); }/mul() { echo $(( $1 * $2 )); }/' lib/calc.sh && make test", "cd /home/eval/work/calc && git add lib/calc.sh && git commit -m 'fix: mul multiplies instead of adding' && git log"]} +{"id": "repo_symlink_release_switch", "steps": ["cd /home/eval/deploy && cat TARGET && ls -l && make test", "cd /home/eval/deploy && ln -sfn releases/v1.5.0 current && make test", "cd /home/eval/deploy && cat current/app.conf && sed -i 's/^port = /port=/' releases/v1.5.0/app.conf && make test", "cd /home/eval/deploy && git add -A && git commit -m 'release: switch live to v1.5.0' && git log -n 1"]} +{"id": "repo_path_bin_tool_fix", "steps": ["cd /home/eval/tools && cat README.md && make test", "cd /home/eval/tools && export PATH=\"$PWD/bin:$PATH\" && make test", "cd /home/eval/tools && sed -i 's/minor) echo \"$major.$((minor + 1)).$patch\"/minor) echo \"$major.$((minor + 1)).0\"/' bin/semver-bump && export PATH=\"$PWD/bin:$PATH\" && make test", "cd /home/eval/tools && git add bin/semver-bump && git commit -m 'fix: minor bump resets patch' && git log -n 1"]} +{"id": "repo_parallel_shards_background", "steps": ["cd /home/eval/etl && cat SHARDS scripts/process.sh", "cd /home/eval/etl && mkdir -p out && declare -A pids && while read -r s; do bash scripts/process.sh \"$s\" & pids[$s]=$!; done < SHARDS; : > out/status.txt; while read -r s; do wait \"${pids[$s]}\"; echo \"$s $?\" >> out/status.txt; done < SHARDS; cat out/status.txt", "cd /home/eval/etl && sed -i '/^B2,n\\/a$/d' data/eu.csv && bash scripts/process.sh eu && make report", "cd /home/eval/etl && git add data out && git commit -m 'data: drop malformed eu row' && git log -n 1"]} +{"id": "repo_report_sort_regression", "steps": ["cd /home/eval/report && make test; cat bin/top-customers", "cd /home/eval/report && sed -i 's/sort -t, -k2 -r/sort -t, -k2,2nr/' bin/top-customers && make test", "cd /home/eval/report && git commit -am 'fix: sort report totals numerically' && git log -n 1"]} +{"id": "repo_jq_config_validation", "steps": ["cd /home/eval/svc && make test; cat build/test_configs.log; cat config/*.json", "cd /home/eval/svc && jq '.port |= tonumber' config/staging.json > /tmp/s.json && mv /tmp/s.json config/staging.json && jq '.env = \"prod\" | .features.audit = true' config/prod.json > /tmp/p.json && mv /tmp/p.json config/prod.json && make test", "cd /home/eval/svc && git add config && git commit -m 'config: fix staging port type and prod env/audit' && git log -n 1"]} +{"id": "repo_release_branch_bump", "steps": ["cd /home/eval/lib && git checkout -b release-1.0.0 && make test; cat package.json", "cd /home/eval/lib && echo 1.0.0 > VERSION && jq '.version = \"1.0.0\"' package.json > package.json.new && mv package.json.new package.json && sed -i 's/^## Unreleased$/## 1.0.0/' CHANGELOG.md && make test", "cd /home/eval/lib && git add . && git commit -m 'release: 1.0.0' && git status"]} +{"id": "repo_parallel_runner_masks_failure", "steps": ["cd /home/eval/parser && make test && make canary; cat tests/run.sh", "cd /home/eval/parser && cat > tests/run.sh <<'EOF'\n#!/bin/bash\n# usage: run.sh DIR -> run DIR/test_*.sh in parallel; exit 1 if any fail\ndir=$1\nmkdir -p build\npids=()\nnames=()\nfor t in \"$dir\"/test_*.sh; do\n bash \"$t\" > \"build/$(basename \"$t\" .sh).log\" 2>&1 &\n pids+=($!)\n names+=(\"$t\")\ndone\nstatus=0\nfor i in \"${!pids[@]}\"; do\n if ! wait \"${pids[$i]}\"; then echo \"FAIL ${names[$i]}\"; status=1; fi\ndone\n[ \"$status\" -eq 0 ] && echo \"all tests passed in $dir\" | tee build/last-run.txt\nexit $status\nEOF\nmake test; make canary", "cd /home/eval/parser && cat build/test_equals_in_value.log && sed -i 's/echo \"${pair##\\*=}\"/echo \"${pair#*=}\"/' src/kv.sh && cat src/kv.sh && make test && make canary", "cd /home/eval/parser && git add -A && git commit -m 'fix: runner propagates job failures; keep = in values' && git log -n 1"]} diff --git a/crates/bashkit-eval/data/repo-workflow.jsonl b/crates/bashkit-eval/data/repo-workflow.jsonl new file mode 100644 index 000000000..45d5b38fd --- /dev/null +++ b/crates/bashkit-eval/data/repo-workflow.jsonl @@ -0,0 +1,8 @@ +{"id": "repo_clone_fix_failing_test", "category": "repo_workflow", "description": "Clone a local fixture repo, run make test, fix the arithmetic bug, commit", "system": null, "prompt": "A git repository lives at /srv/git/calc. Clone it into /home/eval/work/calc, run its test suite with `make test`, find and fix the bug in the library code that makes a test fail (do not edit the tests), re-run `make test` until it passes, and commit the fix in the clone with a commit message that starts with `fix:`. Leave /srv/git/calc unchanged.", "files": {"/srv/git/calc/Makefile": "test:\n\t@bash tests/run.sh\n", "/srv/git/calc/README.md": "# calc\n\nTiny bash arithmetic library. Run `make test`.\n", "/srv/git/calc/lib/calc.sh": "# calc: integer helpers\nadd() { echo $(( $1 + $2 )); }\nsub() { echo $(( $1 - $2 )); }\nmul() { echo $(( $1 + $2 )); }\ndiv() {\n if [ \"$2\" -eq 0 ]; then echo 'calc: division by zero' >&2; return 1; fi\n echo $(( $1 / $2 ))\n}\n", "/srv/git/calc/tests/run.sh": "#!/bin/bash\n# Test runner: runs every tests/test_*.sh, writes build/test-report.txt.\ncd \"$(dirname \"$0\")/..\" || exit 2\nmkdir -p build\npass=0; fail=0; total=0\nfor t in tests/test_*.sh; do\n name=$(basename \"$t\" .sh)\n total=$((total + 1))\n if bash \"$t\" > \"build/$name.log\" 2>&1; then\n pass=$((pass + 1)); echo \"ok $name\"\n else\n fail=$((fail + 1)); echo \"FAIL $name\"; sed 's/^/ /' \"build/$name.log\"\n fi\ndone\nif [ \"$fail\" -eq 0 ]; then\n echo \"PASS $pass/$total\" | tee build/test-report.txt\nelse\n echo \"FAIL $fail/$total\" | tee build/test-report.txt\n exit 1\nfi\n", "/srv/git/calc/tests/assert.sh": "# Minimal assertion helpers for the test suite.\nassert_eq() {\n if [ \"$1\" != \"$2\" ]; then\n echo \"assertion failed: ${3:-values differ}: expected '$1', got '$2'\"\n exit 1\n fi\n}\n", "/srv/git/calc/tests/test_add_sub.sh": ". lib/calc.sh\n. tests/assert.sh\nassert_eq 5 \"$(add 2 3)\" 'add 2 3'\nassert_eq -4 \"$(sub 3 7)\" 'sub 3 7'\n", "/srv/git/calc/tests/test_mul_div.sh": ". lib/calc.sh\n. tests/assert.sh\nassert_eq 20 \"$(mul 4 5)\" 'mul 4 5'\nassert_eq 0 \"$(mul 7 0)\" 'mul 7 0'\nassert_eq 3 \"$(div 9 3)\" 'div 9 3'\ndiv 1 0 2>/dev/null && { echo 'div by zero must fail'; exit 1; }\nexit 0\n"}, "setup": "cd /srv/git/calc\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_line_regex:/home/eval/work/calc/lib/calc.sh:^mul\\(\\).*\\*"}, {"check": "file_contains:/home/eval/work/calc/build/test-report.txt:PASS 2/2"}, {"check": "file_line_regex:/home/eval/work/calc/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|fix:"}, {"check": "file_line_regex:/home/eval/work/calc/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|initial import$"}, {"check": "file_contains:/home/eval/work/calc/tests/test_mul_div.sh:assert_eq 20 \"$(mul 4 5)\""}, {"check": "file_contains:/srv/git/calc/lib/calc.sh:mul() { echo $(( $1 + $2 )); }"}]} +{"id": "repo_symlink_release_switch", "category": "repo_workflow", "description": "Repoint a `current` release symlink, fix the new release config, run checks, commit", "system": null, "prompt": "/home/eval/deploy is a git repo of release directories under releases/, with a `current` symlink pointing at the live one. The file TARGET names the release that should be live. Repoint `current` at that release (keep it a symlink; do not copy or rename release directories), then run `make test`. If the check reports a problem in the target release's files, fix it and re-run until `make test` passes. Commit everything with a message starting with `release:`.", "files": {"/home/eval/deploy/Makefile": "test:\n\t@bash bin/check-release\n", "/home/eval/deploy/TARGET": "v1.5.0\n", "/home/eval/deploy/releases/v1.4.0/VERSION": "1.4.0\n", "/home/eval/deploy/releases/v1.4.0/app.conf": "name=web\nport=8080\nworkers=4\n", "/home/eval/deploy/releases/v1.5.0/VERSION": "1.5.0\n", "/home/eval/deploy/releases/v1.5.0/app.conf": "name=web\nport = 8080\nworkers=6\n", "/home/eval/deploy/bin/check-release": "#!/bin/bash\n# Verify the live release (`current`) is the TARGET release and is well-formed.\nset -u\nmkdir -p build\nwant=$(cat TARGET)\n[ -L current ] || { echo 'current must be a symlink'; exit 1; }\nhave=\"v$(cat current/VERSION)\"\nif [ \"$have\" != \"$want\" ]; then echo \"live release is $have, TARGET is $want\"; exit 1; fi\nfor key in name port workers; do\n grep -Eq \"^$key=[^ ]+$\" current/app.conf || { echo \"app.conf: missing or malformed '$key=VALUE' line\"; exit 1; }\ndone\necho \"OK $want\" | tee build/release-check.txt\n"}, "setup": "cd /home/eval/deploy\nln -s releases/v1.4.0 current\nchmod +x bin/check-release\ncd /home/eval/deploy\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "symlink:/home/eval/deploy/current:releases/v1.5.0"}, {"check": "file_line_regex:/home/eval/deploy/releases/v1.5.0/app.conf:^port=8080$"}, {"check": "file_contains:/home/eval/deploy/releases/v1.4.0/app.conf:port=8080"}, {"check": "file_contains:/home/eval/deploy/build/release-check.txt:OK v1.5.0"}, {"check": "file_line_regex:/home/eval/deploy/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|release:"}]} +{"id": "repo_path_bin_tool_fix", "category": "repo_workflow", "description": "Put the repo's bin/ on PATH, run make test, fix the semver-bump script, commit", "system": null, "prompt": "/home/eval/tools is a git repo with helper scripts in bin/. Its README explains how to run the tests. Get `make test` passing: follow the README's setup, then fix the bug in the script under bin/ that the failing test exposes (do not change the tests or the Makefile). Commit the fix with a message starting with `fix:`.", "files": {"/home/eval/tools/README.md": "# tools\n\nThe tests call the scripts in `bin/` by name, so put that directory on your PATH first:\n\n export PATH=\"$PWD/bin:$PATH\"\n make test\n", "/home/eval/tools/Makefile": "test:\n\t@bash tests/run.sh\n", "/home/eval/tools/bin/semver-bump": "#!/bin/bash\n# usage: semver-bump VERSION major|minor|patch\nIFS=. read -r major minor patch <<< \"$1\"\ncase \"$2\" in\n major) echo \"$((major + 1)).0.0\" ;;\n minor) echo \"$major.$((minor + 1)).$patch\" ;;\n patch) echo \"$major.$minor.$((patch + 1))\" ;;\n *) echo \"semver-bump: unknown part '$2'\" >&2; exit 2 ;;\nesac\n", "/home/eval/tools/bin/slugify": "#!/bin/bash\n# usage: slugify TEXT -> lower-case, dash-separated\necho \"$*\" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+|-+$//g'\n", "/home/eval/tools/tests/run.sh": "#!/bin/bash\n# Test runner: runs every tests/test_*.sh, writes build/test-report.txt.\ncd \"$(dirname \"$0\")/..\" || exit 2\nmkdir -p build\npass=0; fail=0; total=0\nfor t in tests/test_*.sh; do\n name=$(basename \"$t\" .sh)\n total=$((total + 1))\n if bash \"$t\" > \"build/$name.log\" 2>&1; then\n pass=$((pass + 1)); echo \"ok $name\"\n else\n fail=$((fail + 1)); echo \"FAIL $name\"; sed 's/^/ /' \"build/$name.log\"\n fi\ndone\nif [ \"$fail\" -eq 0 ]; then\n echo \"PASS $pass/$total\" | tee build/test-report.txt\nelse\n echo \"FAIL $fail/$total\" | tee build/test-report.txt\n exit 1\nfi\n", "/home/eval/tools/tests/assert.sh": "# Minimal assertion helpers for the test suite.\nassert_eq() {\n if [ \"$1\" != \"$2\" ]; then\n echo \"assertion failed: ${3:-values differ}: expected '$1', got '$2'\"\n exit 1\n fi\n}\n", "/home/eval/tools/tests/test_semver.sh": ". tests/assert.sh\ncommand -v semver-bump >/dev/null || { echo 'semver-bump not on PATH'; exit 1; }\nassert_eq 2.0.0 \"$(semver-bump 1.4.2 major)\" major\nassert_eq 1.5.0 \"$(semver-bump 1.4.2 minor)\" minor\nassert_eq 1.4.3 \"$(semver-bump 1.4.2 patch)\" patch\n", "/home/eval/tools/tests/test_slugify.sh": ". tests/assert.sh\ncommand -v slugify >/dev/null || { echo 'slugify not on PATH'; exit 1; }\nassert_eq hello-big-world \"$(slugify ' Hello, Big World! ')\" slugify\n"}, "setup": "chmod +x /home/eval/tools/bin/semver-bump /home/eval/tools/bin/slugify\ncd /home/eval/tools\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_contains:/home/eval/tools/build/test-report.txt:PASS 2/2"}, {"check": "file_contains:/home/eval/tools/tests/test_semver.sh:\"$(semver-bump 1.4.2 minor)\""}, {"check": "file_contains:/home/eval/tools/Makefile:@bash tests/run.sh"}, {"check": "file_not_contains:/home/eval/tools/bin/semver-bump:$((minor + 1)).$patch"}, {"check": "file_line_regex:/home/eval/tools/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|fix:"}]} +{"id": "repo_parallel_shards_background", "category": "repo_workflow", "description": "Run shard jobs concurrently in the background, record each exit status, fix the bad shard, merge, commit", "system": null, "prompt": "/home/eval/etl is a git repo. `scripts/process.sh SHARD` sums the amount column of data/SHARD.csv into out/SHARD.sum. Run it for the shards listed in SHARDS concurrently as background jobs, then wait for each job and write its exit status to out/status.txt, one line per shard in SHARDS order, formatted `SHARD EXIT_CODE`. One shard will fail: find the bad input row it reports and delete that row from the data file, re-run just that shard, then run `make report`. Commit the data fix and out/ with a message starting with `data:`.", "files": {"/home/eval/etl/SHARDS": "us\neu\napac\n", "/home/eval/etl/Makefile": "report:\n\t@bash scripts/report.sh\n", "/home/eval/etl/scripts/process.sh": "#!/bin/bash\n# usage: process.sh SHARD -> out/SHARD.sum (exit 2 on a malformed row)\nshard=$1\nmkdir -p out\nawk -F, -v shard=\"$shard\" '\n NR == 1 { next }\n $2 !~ /^[0-9]+$/ { printf \"%s: line %d: bad amount %s\\n\", shard, NR, $2 > \"/dev/stderr\"; bad = 1; exit 2 }\n { sum += $2 }\n END { if (!bad) print sum }\n' \"data/$shard.csv\" > \"out/$shard.sum.tmp\" || { rm -f \"out/$shard.sum.tmp\"; exit 2; }\nmv \"out/$shard.sum.tmp\" \"out/$shard.sum\"\n", "/home/eval/etl/scripts/report.sh": "#!/bin/bash\n# Combine out/*.sum (every shard in SHARDS must be present) into out/report.txt\ntotal=0\n: > out/report.txt\nwhile read -r shard; do\n [ -f \"out/$shard.sum\" ] || { echo \"missing out/$shard.sum\"; exit 1; }\n v=$(cat \"out/$shard.sum\")\n echo \"$shard $v\" >> out/report.txt\n total=$((total + v))\ndone < SHARDS\necho \"total $total\" >> out/report.txt\ncat out/report.txt\n", "/home/eval/etl/data/us.csv": "order,amount\nA1,120\nA2,80\nA3,300\n", "/home/eval/etl/data/eu.csv": "order,amount\nB1,50\nB2,n/a\nB3,70\n", "/home/eval/etl/data/apac.csv": "order,amount\nC1,15\nC2,25\n"}, "setup": "cd /home/eval/etl\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_line_regex:/home/eval/etl/out/status.txt:^us 0$"}, {"check": "file_line_regex:/home/eval/etl/out/status.txt:^eu 2$"}, {"check": "file_line_regex:/home/eval/etl/out/status.txt:^apac 0$"}, {"check": "file_not_contains:/home/eval/etl/data/eu.csv:n/a"}, {"check": "file_contains:/home/eval/etl/data/eu.csv:B3,70"}, {"check": "file_line_regex:/home/eval/etl/out/report.txt:^eu 120$"}, {"check": "file_line_regex:/home/eval/etl/out/report.txt:^total 660$"}, {"check": "file_line_regex:/home/eval/etl/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|data:"}]} +{"id": "repo_report_sort_regression", "category": "repo_workflow", "description": "Fix a lexicographic-vs-numeric sort bug in a report script so the golden-file test passes, commit", "system": null, "prompt": "`make test` fails in the git repo /home/eval/report: the top-customers report no longer matches the golden file tests/expected.txt. The golden file is correct. Fix the report script, make sure `make test` passes, and commit with a message starting with `fix:`.", "files": {"/home/eval/report/Makefile": "test:\n\t@mkdir -p build\n\t@bin/top-customers data/orders.csv 3 > build/actual.txt\n\t@diff -u tests/expected.txt build/actual.txt && echo 'report OK'\n", "/home/eval/report/bin/top-customers": "#!/bin/bash\n# usage: top-customers ORDERS.csv N -> top N customers by total spend\nawk -F, 'NR > 1 { total[$2] += $3 } END { for (c in total) print c \",\" total[c] }' \"$1\" \\\n | sort -t, -k2 -r \\\n | head -n \"$2\"\n", "/home/eval/report/data/orders.csv": "id,customer,amount\n1,acme,900\n2,globex,1200\n3,initech,450\n4,acme,300\n5,umbrella,95\n6,globex,40\n7,initech,650\n8,hooli,99\n", "/home/eval/report/tests/expected.txt": "globex,1240\nacme,1200\ninitech,1100\n"}, "setup": "chmod +x /home/eval/report/bin/top-customers\ncd /home/eval/report\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_contains:/home/eval/report/build/actual.txt:globex,1240\nacme,1200\ninitech,1100\n"}, {"check": "file_contains:/home/eval/report/tests/expected.txt:globex,1240\nacme,1200\ninitech,1100\n"}, {"check": "file_line_regex:/home/eval/report/bin/top-customers:sort .*n"}, {"check": "file_line_regex:/home/eval/report/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|fix:"}]} +{"id": "repo_jq_config_validation", "category": "repo_workflow", "description": "Fix per-environment JSON configs until a jq-based validator passes, commit", "system": null, "prompt": "In the git repo /home/eval/svc, `make test` validates every config/*.json file with jq. Make it pass by fixing the config files (not the validator), keeping every value's meaning the same. Then commit with a message starting with `config:`.", "files": {"/home/eval/svc/Makefile": "test:\n\t@bash tests/run.sh\n", "/home/eval/svc/tests/run.sh": "#!/bin/bash\n# Test runner: runs every tests/test_*.sh, writes build/test-report.txt.\ncd \"$(dirname \"$0\")/..\" || exit 2\nmkdir -p build\npass=0; fail=0; total=0\nfor t in tests/test_*.sh; do\n name=$(basename \"$t\" .sh)\n total=$((total + 1))\n if bash \"$t\" > \"build/$name.log\" 2>&1; then\n pass=$((pass + 1)); echo \"ok $name\"\n else\n fail=$((fail + 1)); echo \"FAIL $name\"; sed 's/^/ /' \"build/$name.log\"\n fi\ndone\nif [ \"$fail\" -eq 0 ]; then\n echo \"PASS $pass/$total\" | tee build/test-report.txt\nelse\n echo \"FAIL $fail/$total\" | tee build/test-report.txt\n exit 1\nfi\n", "/home/eval/svc/tests/test_configs.sh": "# Each config: env matches the file name, port is a number, replicas >= 1,\n# and every feature flag is a boolean.\nstatus=0\nfor f in config/*.json; do\n env=$(basename \"$f\" .json)\n jq -e --arg env \"$env\" '\n (.env == $env) and (.port | type == \"number\") and (.replicas >= 1)\n and ([.features[] | type == \"boolean\"] | all)' \"$f\" > /dev/null \\\n || { echo \"invalid: $f\"; status=1; }\ndone\nexit $status\n", "/home/eval/svc/config/dev.json": "{\n \"env\": \"dev\",\n \"port\": 3000,\n \"replicas\": 1,\n \"features\": {\n \"beta\": true,\n \"audit\": false\n }\n}\n", "/home/eval/svc/config/staging.json": "{\n \"env\": \"staging\",\n \"port\": \"8080\",\n \"replicas\": 2,\n \"features\": {\n \"beta\": true,\n \"audit\": true\n }\n}\n", "/home/eval/svc/config/prod.json": "{\n \"env\": \"production\",\n \"port\": 443,\n \"replicas\": 6,\n \"features\": {\n \"beta\": false,\n \"audit\": \"true\"\n }\n}\n"}, "setup": "cd /home/eval/svc\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_line_regex:/home/eval/svc/config/staging.json:\"port\":\\s*8080\\s*,?$"}, {"check": "file_line_regex:/home/eval/svc/config/prod.json:\"env\":\\s*\"prod\"\\s*,?$"}, {"check": "file_line_regex:/home/eval/svc/config/prod.json:\"audit\":\\s*true\\s*,?$"}, {"check": "file_line_regex:/home/eval/svc/config/prod.json:\"replicas\":\\s*6\\s*,?$"}, {"check": "file_contains:/home/eval/svc/build/test-report.txt:PASS 1/1"}, {"check": "file_line_regex:/home/eval/svc/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|config:"}]} +{"id": "repo_release_branch_bump", "category": "repo_workflow", "description": "Cut a release branch, bump the version in three files, run consistency checks, commit", "system": null, "prompt": "Cut release 1.0.0 of the git repo /home/eval/lib: create and switch to a branch named `release-1.0.0`, set the version to 1.0.0 everywhere it appears (VERSION, package.json, and turn the `## Unreleased` heading in CHANGELOG.md into `## 1.0.0`), run `make test`, and commit with the message `release: 1.0.0`.", "files": {"/home/eval/lib/VERSION": "0.9.3\n", "/home/eval/lib/package.json": "{\n \"name\": \"@acme/lib\",\n \"version\": \"0.9.3\",\n \"private\": true\n}\n", "/home/eval/lib/CHANGELOG.md": "# Changelog\n\n## Unreleased\n\n- Add retry helper\n- Drop Node 16\n\n## 0.9.3\n\n- Fix timeout parsing\n", "/home/eval/lib/Makefile": "test:\n\t@bash scripts/check-version.sh\n", "/home/eval/lib/scripts/check-version.sh": "#!/bin/bash\n# VERSION, package.json and the newest CHANGELOG heading must agree.\nv=$(cat VERSION)\np=$(jq -r .version package.json)\nc=$(grep -m1 '^## ' CHANGELOG.md | sed 's/^## //')\nif [ \"$v\" = \"$p\" ] && [ \"$v\" = \"$c\" ]; then\n mkdir -p build; echo \"version $v consistent\" | tee build/version-check.txt\nelse\n echo \"mismatch: VERSION=$v package.json=$p CHANGELOG=$c\"; exit 1\nfi\n"}, "setup": "cd /home/eval/lib\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_contains:/home/eval/lib/.git/HEAD:refs/heads/release-1.0.0"}, {"check": "file_line_regex:/home/eval/lib/VERSION:^1\\.0\\.0$"}, {"check": "file_line_regex:/home/eval/lib/package.json:\"version\":\\s*\"1\\.0\\.0\""}, {"check": "file_line_regex:/home/eval/lib/CHANGELOG.md:^## 1\\.0\\.0$"}, {"check": "file_line_regex:/home/eval/lib/CHANGELOG.md:^## 0\\.9\\.3$"}, {"check": "file_not_contains:/home/eval/lib/CHANGELOG.md:Unreleased"}, {"check": "file_contains:/home/eval/lib/build/version-check.txt:version 1.0.0 consistent"}, {"check": "file_line_regex:/home/eval/lib/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|release: 1\\.0\\.0$"}]} +{"id": "repo_parallel_runner_masks_failure", "category": "repo_workflow", "description": "Fix a background-job test runner that ignores job exit codes, then fix the bug it was hiding, commit", "system": null, "prompt": "In the git repo /home/eval/parser, `make test` runs the tests in parallel and always reports success, but CI says a test is broken. First fix tests/run.sh so a failing test makes `make test` fail (keep the tests running in parallel as background jobs). Then fix the bug in src/ that the failing test exposes, without editing the tests. Finish with `make test` and `make canary` (which must report a non-zero exit), and commit with a message starting with `fix:`.", "files": {"/home/eval/parser/Makefile": "test:\n\t@bash tests/run.sh tests/unit\n\n# The canary suite always contains a failing test: the runner must notice it.\ncanary:\n\t@mkdir -p build\n\t@bash tests/run.sh tests/canary; echo \"canary exit=$$?\" | tee build/canary.txt\n", "/home/eval/parser/tests/run.sh": "#!/bin/bash\n# usage: run.sh DIR -> run DIR/test_*.sh in parallel; exit 1 if any fail\ndir=$1\nmkdir -p build\nfor t in \"$dir\"/test_*.sh; do\n bash \"$t\" > \"build/$(basename \"$t\" .sh).log\" 2>&1 &\ndone\nwait\nstatus=$?\n[ \"$status\" -eq 0 ] && echo \"all tests passed in $dir\" | tee build/last-run.txt\nexit $status\n", "/home/eval/parser/src/kv.sh": "# parse_kv 'k1=v1;k2=v2' KEY -> value for KEY (values may contain '=')\nparse_kv() {\n local IFS=';' pair\n for pair in $1; do\n if [ \"${pair%%=*}\" = \"$2\" ]; then echo \"${pair##*=}\"; return 0; fi\n done\n return 1\n}\n", "/home/eval/parser/tests/unit/test_basic.sh": ". src/kv.sh\n[ \"$(parse_kv 'a=1;b=2' b)\" = 2 ] || { echo 'b should be 2'; exit 1; }\nparse_kv 'a=1' z && { echo 'missing key must fail'; exit 1; }\nexit 0\n", "/home/eval/parser/tests/unit/test_equals_in_value.sh": ". src/kv.sh\ngot=$(parse_kv 'url=https://x.test/?q=1;mode=fast' url)\n[ \"$got\" = 'https://x.test/?q=1' ] || { echo \"url: got '$got'\"; exit 1; }\n", "/home/eval/parser/tests/canary/test_always_fails.sh": "echo 'canary: expected failure'\nexit 1\n", "/home/eval/parser/tests/canary/test_ok.sh": "exit 0\n"}, "setup": "cd /home/eval/parser\ngit init -q\ngit add .\ngit commit -q -m 'initial import'\n", "expectations": [{"check": "file_contains:/home/eval/parser/build/canary.txt:canary exit=1"}, {"check": "file_contains:/home/eval/parser/build/last-run.txt:all tests passed in tests/unit"}, {"check": "file_line_regex:/home/eval/parser/tests/run.sh:&"}, {"check": "file_line_regex:/home/eval/parser/tests/run.sh:wait"}, {"check": "file_contains:/home/eval/parser/tests/unit/test_equals_in_value.sh:[ \"$got\" = 'https://x.test/?q=1' ]"}, {"check": "file_not_contains:/home/eval/parser/src/kv.sh:##*="}, {"check": "file_line_regex:/home/eval/parser/.git/commits:^[0-9a-f]+\\|[^|]*\\|[^|]*\\|[0-9]+\\|fix:"}]} diff --git a/crates/bashkit-eval/src/agent.rs b/crates/bashkit-eval/src/agent.rs index ea2165542..6c3d2a55d 100644 --- a/crates/bashkit-eval/src/agent.rs +++ b/crates/bashkit-eval/src/agent.rs @@ -3,7 +3,7 @@ // BashTool used only for input_schema/system_prompt introspection use anyhow::{Context, Result}; -use bashkit::{Bash, BashTool, Tool}; +use bashkit::{Bash, BashTool, GitConfig, Tool}; use serde::{Deserialize, Serialize}; use crate::dataset::EvalTask; @@ -55,6 +55,44 @@ fn format_tool_output(stdout: &str, stderr: &str, exit_code: i32) -> String { out } +/// Virtual git identity for eval sessions (TM-GIT-002: never the host's). +pub const EVAL_GIT_AUTHOR: (&str, &str) = ("Eval Agent", "eval@bashkit-eval.invalid"); + +/// Build the task's starting `Bash`: mount `files`, enable sandboxed `git`, then +/// run the optional `setup` script (fixture repos, symlinks, exec bits). +/// +/// Decision: `setup` runs inside a subshell so its `cd`/variables never leak +/// into the agent's session, and its output is not recorded as a tool call +/// (checks see only the agent's own calls). A failing setup is an infra error, +/// not a model failure. Shared with the reference-solution test so CI proves +/// every task is solvable on the exact starting state the model sees. +pub async fn build_task_bash(task: &EvalTask) -> Result { + let mut builder = Bash::builder() + .username("eval") + .hostname("bashkit-eval") + .git(GitConfig::new().author(EVAL_GIT_AUTHOR.0, EVAL_GIT_AUTHOR.1)); + + for (path, content) in &task.files { + builder = builder.mount_text(path, content); + } + let mut bash = builder.build(); + + if let Some(setup) = task.setup.as_deref().filter(|s| !s.trim().is_empty()) { + let r = bash + .exec(&format!("(\nset -e\n{setup}\n)")) + .await + .with_context(|| format!("task {} setup failed to run", task.id))?; + anyhow::ensure!( + r.exit_code == 0, + "task {} setup exited {}: {}", + task.id, + r.exit_code, + r.stderr + ); + } + Ok(bash) +} + /// Run the agent loop for a single task. /// Returns (trace, bash) — bash kept for VFS inspection by scorer. pub async fn run_agent_loop( @@ -62,13 +100,7 @@ pub async fn run_agent_loop( task: &EvalTask, max_turns: usize, ) -> Result<(AgentTrace, Bash)> { - // Build Bash with pre-populated files - let mut builder = Bash::builder().username("eval").hostname("bashkit-eval"); - - for (path, content) in &task.files { - builder = builder.mount_text(path, content); - } - let mut bash = builder.build(); + let mut bash = build_task_bash(task).await?; // Get tool definition from BashTool with matching config let tool = BashTool::builder() diff --git a/crates/bashkit-eval/src/checks.rs b/crates/bashkit-eval/src/checks.rs index 823b27f06..eebed4dd5 100644 --- a/crates/bashkit-eval/src/checks.rs +++ b/crates/bashkit-eval/src/checks.rs @@ -10,6 +10,10 @@ // exit_code:N stdout_contains:text stdout_regex:pattern // stderr_empty file_exists:/path dir_exists:/path // file_contains:/path:text file_line_regex:/path:pattern llm_judge:prompt +// file_not_contains:/path:text (file must exist and not contain text) +// symlink:/path:target (/path is a symlink whose target resolves to the +// same absolute path as `target`; relative or absolute +// spellings both pass) use std::collections::BTreeMap; @@ -86,7 +90,9 @@ pub fn evaluate_check( "file_exists" => check_file_exists(check, weight, check_value, snap, files), "dir_exists" => check_dir_exists(check, weight, check_value, snap), "file_contains" => check_file_contains(check, weight, check_value, files), + "file_not_contains" => check_file_not_contains(check, weight, check_value, files), "file_line_regex" => check_file_line_regex(check, weight, check_value, files), + "symlink" => check_symlink(check, weight, check_value, snap), "llm_judge" => CheckResult { check: check.to_string(), passed: true, @@ -251,6 +257,37 @@ fn check_file_contains( } } +fn check_file_not_contains( + check: &str, + weight: f64, + value: &str, + files: &BTreeMap, +) -> CheckResult { + // Format: "file_not_contains:/path:text". A missing file fails: the check + // asserts an edit to a file that must still exist, not its deletion. + let Some((path_str, text)) = value.split_once(':') else { + return CheckResult { + check: check.to_string(), + passed: false, + detail: "invalid format, expected file_not_contains:/path:text".to_string(), + weight, + }; + }; + let (passed, detail) = match files.get(path_str) { + Some(content) if content.contains(text) => { + (false, format!("'{}' still present in {}", text, path_str)) + } + Some(_) => (true, "absent from file".to_string()), + None => (false, format!("cannot read {}", path_str)), + }; + CheckResult { + check: check.to_string(), + passed, + detail, + weight, + } +} + fn check_file_line_regex( check: &str, weight: f64, @@ -309,6 +346,51 @@ fn check_file_line_regex( } } +/// Lexically resolve `target` against the directory holding `link` +/// (`.`/`..` folded, no filesystem access), the way `readlink -m` would. +fn resolve_link_target(link: &str, target: &str) -> String { + let mut parts: Vec<&str> = Vec::new(); + if !target.starts_with('/') { + let parent = link.rsplit_once('/').map(|(p, _)| p).unwrap_or(""); + parts.extend(parent.split('/').filter(|p| !p.is_empty())); + } + for seg in target.split('/') { + match seg { + "" | "." => {} + ".." => { + parts.pop(); + } + s => parts.push(s), + } + } + format!("/{}", parts.join("/")) +} + +fn check_symlink(check: &str, weight: f64, value: &str, snap: &Snapshot) -> CheckResult { + // Format: "symlink:/link/path:target". + let Some((link, expected)) = value.split_once(':') else { + return CheckResult { + check: check.to_string(), + passed: false, + detail: "invalid format, expected symlink:/path:target".to_string(), + weight, + }; + }; + let (passed, detail) = match snap.links.get(link) { + None => (false, format!("{link} is not a symlink")), + Some(actual) => { + let ok = resolve_link_target(link, actual) == resolve_link_target(link, expected); + (ok, format!("{link} -> {actual}")) + } + }; + CheckResult { + check: check.to_string(), + passed, + detail, + weight, + } +} + #[cfg(test)] mod tests { use super::*; @@ -320,6 +402,7 @@ mod tests { tool_outputs: outputs, last_exit_code, dirs: Vec::new(), + links: BTreeMap::new(), } } @@ -379,6 +462,7 @@ mod tests { tool_outputs: vec![], last_exit_code: Some(0), dirs: vec!["/project/src".to_string()], + links: BTreeMap::new(), }; assert!(check_dir_exists("dir_exists:/project/src", 1.0, "/project/src", &snap).passed); assert!(!check_dir_exists("dir_exists:/project/x", 1.0, "/project/x", &snap).passed); @@ -390,6 +474,7 @@ mod tests { tool_outputs: vec![], last_exit_code: Some(0), dirs: vec!["/a/dir".to_string()], + links: BTreeMap::new(), }; let files = files_with(&[("/a/file.txt", "x")]); assert!( @@ -494,6 +579,42 @@ mod tests { } } + #[test] + fn file_not_contains_pass_fail_and_missing() { + let files = files_with(&[("/d/eu.csv", "order,amount\nB1,50\n")]); + let ok = "/d/eu.csv:n/a"; + assert!(check_file_not_contains("c", 1.0, ok, &files).passed); + let present = "/d/eu.csv:B1,50"; + assert!(!check_file_not_contains("c", 1.0, present, &files).passed); + let missing = "/d/gone.csv:n/a"; + assert!(!check_file_not_contains("c", 1.0, missing, &files).passed); + } + + #[test] + fn symlink_accepts_relative_or_absolute_target() { + let mut snap = snap_with(vec![]); + snap.links + .insert("/srv/app/current".into(), "releases/v2".into()); + let v = "/srv/app/current:releases/v2"; + assert!(check_symlink("symlink", 1.0, v, &snap).passed); + let v = "/srv/app/current:/srv/app/releases/v2"; + assert!(check_symlink("symlink", 1.0, v, &snap).passed); + let v = "/srv/app/current:./releases/v2/"; + assert!(check_symlink("symlink", 1.0, v, &snap).passed); + } + + #[test] + fn symlink_rejects_wrong_target_or_non_link() { + let mut snap = snap_with(vec![]); + snap.links + .insert("/srv/app/current".into(), "releases/v1".into()); + let v = "/srv/app/current:releases/v2"; + assert!(!check_symlink("symlink", 1.0, v, &snap).passed); + // A copied directory (no symlink recorded) must not pass. + let v = "/srv/app/other:releases/v2"; + assert!(!check_symlink("symlink", 1.0, v, &snap).passed); + } + #[test] fn evaluate_aggregates_weighted_rate_and_all_passed() { let snap = snap_with(vec![ToolOutput { diff --git a/crates/bashkit-eval/src/dataset.rs b/crates/bashkit-eval/src/dataset.rs index 0427e8168..4994f5c24 100644 --- a/crates/bashkit-eval/src/dataset.rs +++ b/crates/bashkit-eval/src/dataset.rs @@ -19,6 +19,11 @@ pub struct EvalTask { /// Files to pre-populate in VFS. Key: absolute path, Value: content. #[serde(default)] pub files: HashMap, + /// Shell script run after `files` are mounted and before the agent starts + /// (e.g. `git init` a fixture repo, create symlinks, `chmod +x`). Runs in a + /// subshell with `set -e`; not shown to the model, not scored. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub setup: Option, pub expectations: Vec, } diff --git a/crates/bashkit-eval/src/lib.rs b/crates/bashkit-eval/src/lib.rs index 524728abd..07d326636 100644 --- a/crates/bashkit-eval/src/lib.rs +++ b/crates/bashkit-eval/src/lib.rs @@ -10,6 +10,7 @@ pub mod checks; pub mod dataset; pub mod mira_study; pub mod provider; +pub mod reference; pub mod replay; pub mod scripting_agent; pub mod scripting_dataset; diff --git a/crates/bashkit-eval/src/main.rs b/crates/bashkit-eval/src/main.rs index 55e933b28..ec0fab282 100644 --- a/crates/bashkit-eval/src/main.rs +++ b/crates/bashkit-eval/src/main.rs @@ -24,6 +24,11 @@ fn bashkit_smoke() -> Eval { mira_study::smoke_eval() } +#[eval] +fn bashkit_repo() -> Eval { + mira_study::repo_eval() +} + #[eval] fn bashkit_scripting() -> Eval { mira_study::scripting_eval() diff --git a/crates/bashkit-eval/src/mira_study.rs b/crates/bashkit-eval/src/mira_study.rs index 058e3958d..ec5f0e616 100644 --- a/crates/bashkit-eval/src/mira_study.rs +++ b/crates/bashkit-eval/src/mira_study.rs @@ -28,11 +28,15 @@ use crate::provider::{ }; use crate::scripting_agent::{ScriptingTrace, run_baseline_agent, run_scripted_agent}; use crate::scripting_dataset::ScriptingEvalTask; -use crate::snapshot::{Snapshot, SnapshotTargets, ToolOutput, snapshot_fs}; +use crate::snapshot::{Snapshot, SnapshotTargets, ToolOutput, snapshot_fs, snapshot_links}; /// Default agent-turn budget per task (matches the original harness). pub const MAX_TURNS: usize = 10; +/// Turn budget for `bashkit_repo`: repo tasks are multi-step by design +/// (explore, run tests, fix, re-run, commit), so they get twice the default. +pub const REPO_MAX_TURNS: usize = 20; + /// Default model matrix. Each target is gated on its provider's API-key env var, /// so an offline run skips them all (CI stays green) and a keyed run lights up /// the subset whose credentials are present. Select subsets with @@ -161,6 +165,7 @@ fn bash_subject() -> impl Subject { let targets = SnapshotTargets::from_expectations(&expectations); let fs = bash.fs(); let (files, dirs) = snapshot_fs(fs.as_ref(), &targets).await; + let links = snapshot_links(fs.as_ref(), &targets).await; let tool_outputs: Vec = trace .tool_calls @@ -176,6 +181,7 @@ fn bash_subject() -> impl Subject { tool_outputs, last_exit_code: trace.last_tool_response.as_ref().map(|r| r.exit_code), dirs, + links, }; let ok = trace.tool_calls.iter().filter(|t| t.exit_code == 0).count(); @@ -287,6 +293,7 @@ fn scripting_transcript(trace: ScriptingTrace) -> Transcript { tool_outputs, last_exit_code, dirs: Vec::new(), + links: Default::default(), }; let ok = trace @@ -387,6 +394,7 @@ fn parse_jsonl(text: &str) -> Vec { /// Embedded datasets (no runtime path dependence — robust under any cwd). const EVAL_TASKS: &str = include_str!("../data/eval-tasks.jsonl"); const SMOKE_TASKS: &str = include_str!("../data/smoke-test.jsonl"); +const REPO_TASKS: &str = include_str!("../data/repo-workflow.jsonl"); const SCRIPTING_MANY_TOOLS: &str = include_str!("../data/scripting-tool/many-tools.jsonl"); const SCRIPTING_DISCOVERY: &str = include_str!("../data/scripting-tool/discovery.jsonl"); const SCRIPTING_PAGINATED: &str = include_str!("../data/scripting-tool/paginated.jsonl"); @@ -423,6 +431,23 @@ pub fn smoke_eval() -> Eval { b.build() } +/// Repo-shaped, multi-turn tasks (`repo_workflow`): a fixture git repo built +/// by the task's `setup`, then run `make test`, fix, commit. Kept out of +/// `bashkit_bash` so its 58-task scores stay comparable across runs. Every +/// task has a reference solution checked in CI (`reference.rs`). +pub fn repo_eval() -> Eval { + let mut b = Eval::new("bashkit_repo") + .describe("Multi-turn repo workflows: make test, fix, commit (symlinks, PATH, jobs, git)") + .subject(bash_subject()) + .scorer(expectations_scorer()) + .max_turns(REPO_MAX_TURNS) + .targets(default_targets()); + for sample in bash_samples(REPO_TASKS) { + b = b.add_sample(sample); + } + b.build() +} + /// The scripting-tool orchestration eval. The `mode` axis compares composing /// mock tools into one `ScriptedTool` (`scripted`) against exposing each tool /// individually (`baseline`). Select with `mira run --axis mode=scripted`. @@ -473,6 +498,17 @@ mod tests { assert_eq!(bash_samples(SMOKE_TASKS).len(), 3); } + #[test] + fn repo_samples_load_with_setup() { + let samples = bash_samples(REPO_TASKS); + assert_eq!(samples.len(), 8); + for s in &samples { + let task: EvalTask = serde_json::from_value(s.metadata["task"].clone()).unwrap(); + assert_eq!(task.category, "repo_workflow", "{}", s.id); + assert!(task.setup.is_some(), "{} has no fixture setup", s.id); + } + } + #[test] fn scripting_samples_load_all_datasets() { let datasets = [ @@ -494,6 +530,7 @@ mod tests { // Smoke: the eval builders produce valid Evals without panicking. let _ = bash_eval(); let _ = smoke_eval(); + let _ = repo_eval(); let _ = scripting_eval(); } } diff --git a/crates/bashkit-eval/src/reference.rs b/crates/bashkit-eval/src/reference.rs new file mode 100644 index 000000000..4cf100c7b --- /dev/null +++ b/crates/bashkit-eval/src/reference.rs @@ -0,0 +1,155 @@ +// Reference solutions: prove dataset tasks are solvable without an LLM. +// +// Decision: a reference solution is a list of `steps`, each one bash tool call, +// replayed in order on the exact starting `Bash` the model gets +// (`agent::build_task_bash`: files + git + setup). Each step becomes a +// `ToolOutput`, so the checks score it exactly like a model run (multi-turn +// shape included: `exit_code` reads the last step, `stdout_contains` any step). +// CI runs every reference (see tests below), so a dataset edit or a bashkit +// regression that makes a task unsolvable fails `cargo test -p bashkit-eval`. +// The setup-only state must NOT pass, so a task cannot be trivially green. +// See knowledge/operations/eval.md ("repo_workflow"). + +use anyhow::{Context, Result}; +use serde::Deserialize; + +use crate::agent::build_task_bash; +use crate::checks::{CheckSummary, evaluate}; +use crate::dataset::EvalTask; +use crate::snapshot::{Snapshot, SnapshotTargets, ToolOutput, snapshot_fs, snapshot_links}; + +/// One task's reference solution (`data/*-solutions.jsonl`). +#[derive(Debug, Clone, Deserialize)] +pub struct ReferenceSolution { + pub id: String, + /// Bash tool calls, in order (one per agent turn). + pub steps: Vec, +} + +/// Run `steps` against the task's starting state and score the result. +pub async fn run_reference(task: &EvalTask, steps: &[String]) -> Result<(CheckSummary, Snapshot)> { + let mut bash = build_task_bash(task).await?; + let mut tool_outputs = Vec::with_capacity(steps.len()); + for step in steps { + let (stdout, stderr, exit_code) = match bash.exec(step).await { + Ok(r) => (r.stdout.to_string(), r.stderr.to_string(), r.exit_code), + Err(e) => (String::new(), e.to_string(), 1), + }; + tool_outputs.push(ToolOutput { + commands: step.clone(), + stdout, + stderr, + exit_code, + }); + } + + let expectations: Vec<(String, f64)> = task + .expectations + .iter() + .map(|e| (e.check.clone(), e.weight)) + .collect(); + let targets = SnapshotTargets::from_expectations(&expectations); + let fs = bash.fs(); + let (files, dirs) = snapshot_fs(fs.as_ref(), &targets).await; + let links = snapshot_links(fs.as_ref(), &targets).await; + let snap = Snapshot { + last_exit_code: tool_outputs.last().map(|t| t.exit_code), + tool_outputs, + dirs, + links, + }; + Ok((evaluate(&expectations, &snap, &files), snap)) +} + +/// Parse a solutions JSONL (blank and `#`/`//` lines skipped). +pub fn parse_solutions(jsonl: &str) -> Result> { + jsonl + .lines() + .enumerate() + .map(|(i, l)| (i, l.trim())) + .filter(|(_, l)| !l.is_empty() && !l.starts_with('#') && !l.starts_with("//")) + .map(|(i, l)| serde_json::from_str(l).with_context(|| format!("solutions line {}", i + 1))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + const REPO_TASKS: &str = include_str!("../data/repo-workflow.jsonl"); + const REPO_SOLUTIONS: &str = include_str!("../data/repo-workflow-solutions.jsonl"); + + fn tasks() -> Vec { + REPO_TASKS + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).expect("task parses")) + .collect() + } + + fn report(summary: &CheckSummary, snap: &Snapshot) -> String { + let mut out = String::new(); + for r in &summary.results { + out.push_str(&format!( + " [{}] {} -- {}\n", + if r.passed { "ok" } else { "FAIL" }, + r.check, + r.detail + )); + } + for t in &snap.tool_outputs { + out.push_str(&format!( + "--- step (exit {}) ---\n$ {}\nstdout:\n{}stderr:\n{}\n", + t.exit_code, t.commands, t.stdout, t.stderr + )); + } + out + } + + #[test] + fn every_task_has_exactly_one_solution() { + let ids: BTreeSet = tasks().into_iter().map(|t| t.id).collect(); + let sols = parse_solutions(REPO_SOLUTIONS).unwrap(); + let sol_ids: BTreeSet = sols.iter().map(|s| s.id.clone()).collect(); + assert_eq!(sols.len(), sol_ids.len(), "duplicate solution ids"); + assert_eq!(ids, sol_ids, "tasks and reference solutions differ"); + assert!(ids.len() >= 6, "repo_workflow needs at least 6 tasks"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn reference_solutions_pass_all_checks() { + let sols = parse_solutions(REPO_SOLUTIONS).unwrap(); + let mut failures = Vec::new(); + for task in tasks() { + let sol = sols.iter().find(|s| s.id == task.id).unwrap(); + let (summary, snap) = run_reference(&task, &sol.steps).await.unwrap(); + if !summary.all_passed() { + failures.push(format!("== {} ==\n{}", task.id, report(&summary, &snap))); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn untouched_fixture_does_not_pass() { + // Only the agent's natural first look (`make test`), no fix: the task + // must still fail, so a do-nothing model cannot score. + for task in tasks() { + let dir = task + .prompt + .split_whitespace() + .find(|w| w.starts_with("/home/eval/")) + .map(|w| w.trim_end_matches(['.', ',', ':', ')'])) + .unwrap_or("/home/eval"); + let steps = vec![format!("cd {dir} 2>/dev/null; make test; git status")]; + let (summary, snap) = run_reference(&task, &steps).await.unwrap(); + assert!( + !summary.all_passed(), + "{} passes without a fix:\n{}", + task.id, + report(&summary, &snap) + ); + } + } +} diff --git a/crates/bashkit-eval/src/snapshot.rs b/crates/bashkit-eval/src/snapshot.rs index 5ac669369..cc0acb7a0 100644 --- a/crates/bashkit-eval/src/snapshot.rs +++ b/crates/bashkit-eval/src/snapshot.rs @@ -36,6 +36,10 @@ pub struct Snapshot { pub last_exit_code: Option, /// Absolute paths of expectation-relevant directories in the VFS after the run. pub dirs: Vec, + /// Expectation-relevant symlinks: link path -> raw `readlink` target. + /// Paths that are not symlinks are absent. + #[serde(default)] + pub links: BTreeMap, } /// Metadata key under which the snapshot rides on the Transcript. @@ -63,6 +67,7 @@ pub const MAX_SNAPSHOT_FILE_BYTES: usize = 1024 * 1024; pub struct SnapshotTargets { pub files: BTreeSet, pub dirs: BTreeSet, + pub links: BTreeSet, } impl SnapshotTargets { @@ -76,10 +81,15 @@ impl SnapshotTargets { targets.files.insert(check_value.to_string()); targets.dirs.insert(check_value.to_string()); } + "symlink" => { + if let Some((path, _)) = check_value.split_once(':') { + targets.links.insert(path.to_string()); + } + } "dir_exists" => { targets.dirs.insert(check_value.to_string()); } - "file_contains" | "file_line_regex" => { + "file_contains" | "file_not_contains" | "file_line_regex" => { if let Some((path, _)) = check_value.split_once(':') { targets.files.insert(path.to_string()); } @@ -126,6 +136,28 @@ pub async fn snapshot_fs( (files, dirs) } +/// Read the raw target of each expectation-relevant symlink (`lstat`, so a +/// link to a directory is reported as the link, not followed). +pub async fn snapshot_links( + fs: &dyn FileSystem, + targets: &SnapshotTargets, +) -> BTreeMap { + let mut links = BTreeMap::new(); + for path in &targets.links { + let path_buf = PathBuf::from(path); + let Ok(metadata) = fs.lstat(&path_buf).await else { + continue; + }; + if metadata.file_type != FileType::Symlink { + continue; + } + if let Ok(target) = fs.read_link(&path_buf).await { + links.insert(path.clone(), target.to_string_lossy().into_owned()); + } + } + links +} + fn lossy_truncated(bytes: &[u8]) -> String { let capped = bytes.len().min(MAX_SNAPSHOT_FILE_BYTES); let mut content = String::from_utf8_lossy(&bytes[..capped]).into_owned(); diff --git a/justfile b/justfile index 75b665658..27ccf303f 100644 --- a/justfile +++ b/justfile @@ -309,6 +309,11 @@ eval *ARGS: eval-smoke *ARGS: mira --bin bashkit-eval run bashkit_smoke {{ARGS}} +# Repo-workflow eval: multi-turn fixture repos (make test, fix, commit). Every +# task's reference solution also runs offline in `cargo test -p bashkit-eval`. +eval-repo *ARGS: + mira --bin bashkit-eval run bashkit_repo {{ARGS}} + # Scripting-tool eval. The `mode` axis compares scripted vs baseline; select one # with `--axis mode=scripted` (omit to run both). eval-scripting *ARGS: diff --git a/knowledge/log.md b/knowledge/log.md index 97f7bd174..1f81056b7 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-08 +* **Test**: `bashkit_repo` eval (`just eval-repo`), category `repo_workflow`: 8 multi-turn repo tasks (clone, `make test`, fix, commit) that combine symlinks, PATH, background jobs, jq/awk/sed and git. Tasks gain an optional `setup` script (fixture git repos) and the eval `Bash` enables sandboxed git; new checks `file_not_contains` and `symlink`. Every task has a reference solution that `cargo test -p bashkit-eval` replays through the real checks (and the untouched fixture must fail), so CI keeps tasks solvable. Gaps the references found, fixed: `ln` gains `-n`/`-T`/`-v`, long options and GNU directory destinations (`ln -sf X current` with `current` a link to a dir now lands inside it, `ln -sfn` replaces it); `mkdir` and `ln -s` under a symlinked directory created literal `link/name` entries (InMemoryFs now requires a directory parent, overlay copies the parent up); an overlay rename (so `sed -i` on a `mount_text` file) reset the mode to 0644; `git add -A`/`--all`/`-u` staged nothing and `git commit -am` failed; `git clone PATH` (local, in-VFS) now copies the repo. Remaining git gaps in integrations/git-support. See operations/eval "Repo Workflow Eval". + * **Test**: Oils spec pass rate. `just oils-spec` (`scripts/oils-spec/run.py`) fetches the upstream Oils spec suite at a pinned commit (`57d3f0d088c3`, not vendored), runs every case of the 132 bash-column spec files through bashkit (`-c`, `realfs` mounts) and real bash (the Oils way), and scores the bash column of each case with Oils' own assertion rules (`## OK/N-I/BUG bash` overrides win). First run: bashkit passes 1997 of the 2693 cases real bash 5.2.21 passes, **74.2%** (2003/2759 raw); bash itself passes 97.6%. Results go to `scripts/oils-spec/results/` and the `/benches` page. Python 2 helpers (`argv.py`, ...) are re-implemented in bash under `scripts/oils-spec/bin/`. See operations/oils-spec. * **Fix**: field splitting splits only expansion results. Literal text beside an unquoted expansion stays whole (`IFS=:; w=a:; ${w}:b` is `a` `:b`, it was `a` `` `b`), an empty quoted part keeps a field (`$space""`, `""$A""` give empty fields), single-quoted text inside a word is protected (`$x'a b'`), and quoted text inside a used `${v:-...}`/`${v:+...}` operand is protected while its unquoted blanks split (`1${u:-"2 3" "4 5"}6` is `12 3` `4 56`; also closes the `${x:-'a b'}` gap). Unquoted `$*`/`$@` join on the first IFS character and split, so a non-whitespace IFS keeps empty parameters (`IFS=x; set -- '' ''; $*`) and a null IFS keeps one field per non-empty parameter; an empty unquoted result under a null IFS makes no field. `${#v}` splits, and `${!a[*]}` / `${!prefix*}` are told apart from their `@` forms (AST `ArrayIndices`/`PrefixMatch` carry `star`). Arguments shaped like `=x` are no longer exempt from splitting (only `name=`). `"a"$((i++))`, `"a"$(cmd)` and `"a"${x:-y z}` read their expansion whole (`"a"$((k++))` printed `a$`). `[\\]_` is a bracket holding a backslash (it kept both backslashes), and `*\(\)` / `*'()'` no longer turn into an extglob. * **Fix**: `GLOBIGNORE` is honored: matches whose path matches a pattern are dropped (FNM_PATHNAME, `:` inside brackets kept), `.` and `..` always go, a non-empty value turns `dotglob` on, and a list emptied this way is a no-match (literal word, `nullglob`, `failglob`). See limitations "`GLOBIGNORE`". diff --git a/knowledge/operations/eval.md b/knowledge/operations/eval.md index 14e546a4f..4e88e78d3 100644 --- a/knowledge/operations/eval.md +++ b/knowledge/operations/eval.md @@ -96,13 +96,23 @@ Unchanged from the original harness. One JSON object per line: ``` `system` = optional system-message override (null = BashTool default); -`expectations` = checks with optional weight (default 1.0). +`expectations` = checks with optional weight (default 1.0); +`setup` = optional shell script run after `files` are mounted and before the +agent starts (build a fixture git repo, create symlinks, `chmod +x`). It runs +in a subshell with `set -e`, its output is not a tool call, and a non-zero exit +is an infra error. The eval `Bash` has the sandboxed `git` builtin enabled +(author `Eval Agent `, TM-GIT-002). Both the agent +loop and the reference-solution test build the starting state with +`agent::build_task_bash`. ## Expectation Check Types `exit_code:N`, `stdout_contains:text`, `stdout_regex:pattern`, `stderr_empty`, `file_exists:/path`, `dir_exists:/path`, -`file_contains:/path:text`, `file_line_regex:/path:pattern`, +`file_contains:/path:text`, `file_not_contains:/path:text` (file must exist), +`file_line_regex:/path:pattern`, `symlink:/path:target` (`/path` is a symlink; +its target resolves lexically to the same absolute path as `target`, so +`releases/v2` and `/abs/releases/v2` both pass and a copied directory fails), `llm_judge:prompt` (stub, weight forced to 0). Semantics in `crates/bashkit-eval/src/checks.rs` (ported verbatim from the original scorer, with byte-for-byte the same pass/fail logic). @@ -126,6 +136,7 @@ Three evals are advertised (`#[eval]` wrappers in `src/main.rs`): |------|---------|-------| | `bashkit_bash` | 58 tasks across 15 categories | Samples tagged by category; select with `--tag ` | | `bashkit_smoke` | 3 tasks | Quick verification | +| `bashkit_repo` | 8 `repo_workflow` tasks | Multi-turn fixture repos; 20-turn budget (`REPO_MAX_TURNS`) | | `bashkit_scripting` | scripting-tool tasks | `mode` axis: `scripted` vs `baseline` | ## CLI @@ -141,8 +152,8 @@ mira --bin bashkit-eval run --format html --out report.html mira --bin bashkit-eval run --resume ``` -`just eval`, `just eval-smoke`, `just eval-scripting`, and `just eval-list` -wrap these. +`just eval`, `just eval-smoke`, `just eval-repo`, `just eval-scripting`, and +`just eval-list` wrap these. ## Output / Metrics @@ -169,6 +180,46 @@ text processing, pipelines, scripting, data transformation, error recovery, system info, archives, JSON processing, complex multi-step tasks, code search, and environment handling, each with task-appropriate pre-populated seed files. +## Repo Workflow Eval (`bashkit_repo`) + +Roadmap step "harder evals": repo-shaped, multi-turn tasks in +`data/repo-workflow.jsonl` (category `repo_workflow`). Each task's `setup` +turns its `files` into a git repo (`git init` + initial commit), then the model +explores, runs `make test`, fixes code/data/config, re-runs, and commits. The +tasks exercise the features agents combine in real repos, together: + +| Task | Exercises | +|------|-----------| +| `repo_clone_fix_failing_test` | local `git clone`, `make test`, bug fix, commit, upstream untouched | +| `repo_symlink_release_switch` | `ln -sfn` on a link to a directory, config fix, `git add -A` | +| `repo_path_bin_tool_fix` | `export PATH=$PWD/bin:$PATH`, exec bits kept across `sed -i` | +| `repo_parallel_shards_background` | `cmd &`, `$!`, per-job `wait PID` exit codes, data fix, `make` target | +| `repo_report_sort_regression` | awk/sort pipeline vs golden file, `git commit -am` | +| `repo_jq_config_validation` | jq-driven validator, typed JSON edits | +| `repo_release_branch_bump` | `git checkout -b`, version bump across VERSION/package.json/CHANGELOG | +| `repo_parallel_runner_masks_failure` | runner that `wait`s without a PID hides failures; fix it, then the bug | + +Python-dependent tasks are out of scope (bash, awk, sed, jq, make only). +Checks are deterministic: final file contents, `symlink:`, `.git/HEAD`, and +`.git/commits` lines (`hash|author|email|ts|message`, the sandboxed git's +storage format) for "a commit with message X exists" and "history kept". Each +task also checks the model did not edit tests/golden files instead of code. + +**Reference solutions** (`data/repo-workflow-solutions.jsonl`, `src/reference.rs`): +one list of bash tool calls per task, replayed on the exact starting `Bash` +(`build_task_bash`) and scored by the same checks as a model run. `cargo test +-p bashkit-eval` requires every reference to pass all checks and the +untouched fixture (`make test; git status` only) to fail, so CI keeps every +task solvable and non-trivial without an LLM. Every reference was also run +against real bash 5.2 + git and gives the same file results. Building the +references found and fixed five bashkit gaps: `ln -n`/`-T`/directory +destinations, `mkdir`/`ln -s` under a symlinked directory, mode loss when +`sed -i` renames over a `mount_text` file, `git add -A` / `commit -am`, and +local-path `git clone` (see [Git Support](../integrations/git-support.md)). + +`bashkit-replay` looks up mira tasks in `eval-tasks.jsonl` only and runs no +`setup`, so `bashkit_repo` runs are not in the gap corpus yet. + ## Scripting-Tool Eval A second eval (`bashkit_scripting`) tests `ScriptedTool` orchestration (see