diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index bb5409d59..16809f9f3 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -1487,8 +1487,14 @@ pub struct Interpreter { /// Next virtual FD to assign for coproc read ends (starts at 63, like bash). coproc_next_fd: i32, /// Persistent fd output table set by `exec N>/path` redirections. - /// Maps fd number to its output target. Used by `>&N` redirections. + /// Maps fd number to its output target. Used by `>&N` redirections; + /// entries for fd 1 and 2 (`exec >log 2>&1`) route the shell's own + /// output at each top-level command (`route_exec_output`). exec_fd_table: HashMap, + /// Output written to a saved copy of the original stdout/stderr + /// (`exec 3>&1 >log; echo hi >&3`) while fd 1/2 are redirected by + /// `exec`. It skips that routing and goes straight to the caller. + exec_passthrough: (crate::StreamData, crate::StreamData), /// Temporary buffer for fd3+ output during compound body execution. /// Populated by `1>&N` (N>=3) in apply_redirections, consumed by /// apply_redirections_fd_table for compound redirect routing. @@ -2031,6 +2037,7 @@ impl Interpreter { coproc_buffers: HashMap::new(), coproc_next_fd: 63, exec_fd_table: HashMap::new(), + exec_passthrough: Default::default(), pending_fd_output: HashMap::new(), pending_fd_targets: Vec::new(), pending_fd_capture_depth: 0, @@ -2188,6 +2195,7 @@ impl Interpreter { coproc_buffers: HashMap::new(), coproc_next_fd: self.coproc_next_fd, exec_fd_table: self.exec_fd_table.clone(), + exec_passthrough: Default::default(), pending_fd_output: HashMap::new(), pending_fd_targets: Vec::new(), pending_fd_capture_depth: 0, @@ -3055,6 +3063,27 @@ impl Interpreter { if self.output_emit_count != emit_count_before { return false; } + // `exec >log` / `exec 2>/dev/null`: the shell's own output is routed + // at the top-level command (`route_exec_output`), not streamed. A + // pipeline stage's sink is a pipe, never the caller. + let (stdout, stderr) = if self.pipe_out.is_none() { + let empty = crate::StreamData::new(); + ( + if self.exec_fd_table.contains_key(&1) { + empty.clone() + } else { + stdout.clone() + }, + if self.exec_fd_table.contains_key(&2) { + empty + } else { + stderr.clone() + }, + ) + } else { + (stdout.clone(), stderr.clone()) + }; + let (stdout, stderr) = (&stdout, &stderr); let stdout_remaining = self .limits @@ -3199,6 +3228,10 @@ impl Interpreter { for command in &script.commands { self.check_cancelled()?; let emit_before = self.output_emit_count; + let emitted_before = ( + self.output_stream_stdout_bytes, + self.output_stream_stderr_bytes, + ); let mut result = self.execute_command(command).await?; if top_level { // Background jobs that finished meanwhile report here, as @@ -3208,7 +3241,13 @@ impl Interpreter { result.stderr.append(&err); } self.check_cancelled()?; + if top_level { + self.route_exec_output(&mut result, emitted_before).await?; + } self.maybe_emit_output(&result.stdout, &result.stderr, emit_before); + if top_level { + self.flush_exec_passthrough(&mut result); + } // Accumulate stdout with truncation if !stdout_truncated { @@ -7251,7 +7290,15 @@ impl Interpreter { } } RedirectKind::Output | RedirectKind::Clobber => { - let fd = redirect.fd.or(resolved_fd_var).unwrap_or(1); + // WTF: `exec {var}>file` (allocate a free fd into var) is not + // implemented; an unset var must not fall back to fd 1. + let Some(fd) = redirect + .fd + .or(resolved_fd_var) + .or(redirect.fd_var.is_none().then_some(1)) + else { + continue; + }; let target_path = self.expand_word(&redirect.target).await?; let path = self.resolve_path(&target_path); self.ensure_persistent_fd_capacity(fd)?; @@ -7261,14 +7308,31 @@ impl Interpreter { } else if is_dev_null(&path) { self.exec_fd_table.insert(fd, FdTarget::DevNull); } else { - // Truncate file on open (like real exec >file) - let _ = self.fs.write_file(&path, b"").await; + // Truncate file on open (like real exec >file); a + // failed open leaves the fd as it was. + if let Err(e) = self.fs.write_file(&path, b"").await { + return Ok(ExecResult::err( + format!( + "bash: {target_path}: {}\n", + crate::error::io_error_reason(&e) + ), + 1, + )); + } self.exec_fd_table .insert(fd, FdTarget::WriteFile(path, target_path)); } } RedirectKind::Append => { - let fd = redirect.fd.or(resolved_fd_var).unwrap_or(1); + // WTF: `exec {var}>file` (allocate a free fd into var) is not + // implemented; an unset var must not fall back to fd 1. + let Some(fd) = redirect + .fd + .or(resolved_fd_var) + .or(redirect.fd_var.is_none().then_some(1)) + else { + continue; + }; let target_path = self.expand_word(&redirect.target).await?; let path = self.resolve_path(&target_path); self.ensure_persistent_fd_capacity(fd)?; @@ -7278,13 +7342,30 @@ impl Interpreter { } else if is_dev_null(&path) { self.exec_fd_table.insert(fd, FdTarget::DevNull); } else { + if let Err(e) = self.fs.append_file(&path, b"").await { + return Ok(ExecResult::err( + format!( + "bash: {target_path}: {}\n", + crate::error::io_error_reason(&e) + ), + 1, + )); + } self.exec_fd_table .insert(fd, FdTarget::AppendFile(path, target_path)); } } RedirectKind::DupOutput => { let target = self.expand_word(&redirect.target).await?; - let fd = redirect.fd.or(resolved_fd_var).unwrap_or(1); + // WTF: `exec {var}>file` (allocate a free fd into var) is not + // implemented; an unset var must not fall back to fd 1. + let Some(fd) = redirect + .fd + .or(resolved_fd_var) + .or(redirect.fd_var.is_none().then_some(1)) + else { + continue; + }; if target == "-" || target == "&-" { // exec N>&- closes the fd self.exec_fd_table.remove(&fd); @@ -7298,13 +7379,88 @@ impl Interpreter { _ => {} } } + // fd 1/2 pointed back at the original streams (`exec 1>&3`) is no + // redirect at all. + if matches!(self.exec_fd_table.get(&1), Some(FdTarget::Stdout)) { + self.exec_fd_table.remove(&1); + } + if matches!(self.exec_fd_table.get(&2), Some(FdTarget::Stderr)) { + self.exec_fd_table.remove(&2); + } let result = ExecResult::default(); self.apply_redirections(result, redirects).await } /// Target for `exec N>&M` / `exec N>/dev/fd/M`. + /// Send a top-level command's output where `exec` pointed fd 1 and 2 + /// (`exec >log 2>&1`). Output a sub-call already streamed to the caller + /// (before the `exec` ran) stays; the rest goes to the target. + // WTF: output written in the same top-level command before `exec >log` + // ran, and not streamed, also goes to the log; routing happens per + // top-level command, not per write. + async fn route_exec_output( + &mut self, + result: &mut ExecResult, + emitted_before: (usize, usize), + ) -> Result<()> { + if !self.exec_fd_table.contains_key(&1) && !self.exec_fd_table.contains_key(&2) { + return Ok(()); + } + let streamed_out = self.output_stream_stdout_bytes - emitted_before.0; + let streamed_err = self.output_stream_stderr_bytes - emitted_before.1; + let take = |data: &mut crate::StreamData, keep: usize| { + let keep = keep.min(data.len()); + let rest = crate::StreamData::from(&data.as_bytes()[keep..]); + *data = data.prefix(keep); + rest + }; + let mut moved = Vec::new(); + if let Some(target) = self.exec_fd_table.get(&1).cloned() { + moved.push((take(&mut result.stdout, streamed_out), target)); + } + if let Some(target) = self.exec_fd_table.get(&2).cloned() { + moved.push((take(&mut result.stderr, streamed_err), target)); + } + for (data, target) in moved { + if data.is_empty() { + continue; + } + match target { + FdTarget::Stdout => result.stdout.append(&data), + FdTarget::Stderr => result.stderr.append(&data), + FdTarget::DevNull => {} + FdTarget::WriteFile(path, _) | FdTarget::AppendFile(path, _) => { + self.fs.append_file(&path, data.as_bytes()).await?; + } + } + } + Ok(()) + } + + /// Hand output written to a saved original stdout/stderr (`>&3` after + /// `exec 3>&1 >log`) to the caller, unrouted. + fn flush_exec_passthrough(&mut self, result: &mut ExecResult) { + let (out, err) = std::mem::take(&mut self.exec_passthrough); + if out.is_empty() && err.is_empty() { + return; + } + if let Some(cb) = self.output_callback.as_mut() { + cb(&out, &err); + self.output_emit_count += 1; + self.output_stream_stdout_bytes += out.len(); + self.output_stream_stderr_bytes += err.len(); + } + result.stdout.append(&out); + result.stderr.append(&err); + } + fn exec_fd_alias_target(&self, target_fd: i32) -> FdTarget { - if target_fd == 1 { + // `exec 3>&1` copies where fd 1 points now (a file after `exec >log`). + if let Some(target) = self.exec_fd_table.get(&target_fd) + && matches!(target_fd, 1 | 2) + { + target.clone() + } else if target_fd == 1 { FdTarget::Stdout } else if target_fd == 2 { FdTarget::Stderr diff --git a/crates/bashkit/src/interpreter/redirection.rs b/crates/bashkit/src/interpreter/redirection.rs index b4f64fbb8..613d866a8 100644 --- a/crates/bashkit/src/interpreter/redirection.rs +++ b/crates/bashkit/src/interpreter/redirection.rs @@ -285,14 +285,27 @@ impl Interpreter { src_fd: i32, target_fd: i32, ) -> Result<()> { - // Check exec_fd_table for persistent fd targets - if let Some(fd_target) = self.exec_fd_table.get(&target_fd).cloned() { + // Check exec_fd_table for persistent fd targets. Fd 1 and 2 are the + // current streams here: `exec >log` is applied where the shell's + // own output leaves (top level), so `$(cmd 2>&1)` still captures. + if target_fd >= 3 + && let Some(fd_target) = self.exec_fd_table.get(&target_fd).cloned() + { let data = if src_fd == 2 { std::mem::take(&mut result.stderr) } else { std::mem::take(&mut result.stdout) }; + // A saved copy of the original stream (`exec 3>&1 >log`) skips + // the `exec` routing of fd 1/2. + let passthrough = match &fd_target { + FdTarget::Stdout => self.exec_fd_table.contains_key(&1), + FdTarget::Stderr => self.exec_fd_table.contains_key(&2), + _ => false, + }; match &fd_target { + FdTarget::Stdout if passthrough => self.exec_passthrough.0.append(&data), + FdTarget::Stderr if passthrough => self.exec_passthrough.1.append(&data), FdTarget::Stdout => result.stdout.append(&data), FdTarget::Stderr => result.stderr.append(&data), FdTarget::DevNull => {} @@ -496,8 +509,11 @@ impl Interpreter { fd1: &mut FdTarget, fd2: &mut FdTarget, ) { - // Look up exec_fd_table for persistent fd targets - if let Some(exec_target) = self.exec_fd_table.get(&target_fd).cloned() { + // Look up exec_fd_table for persistent fd targets (fd 1/2: see + // `dup_output_fast`). + if target_fd >= 3 + && let Some(exec_target) = self.exec_fd_table.get(&target_fd).cloned() + { match src_fd { 2 => *fd2 = exec_target, _ => *fd1 = exec_target, diff --git a/crates/bashkit/tests/integration/pipeline_tests.rs b/crates/bashkit/tests/integration/pipeline_tests.rs index 2ae3e4fab..c296d190e 100644 --- a/crates/bashkit/tests/integration/pipeline_tests.rs +++ b/crates/bashkit/tests/integration/pipeline_tests.rs @@ -163,3 +163,26 @@ async fn redirected_function_output_is_not_streamed() { assert_eq!(result.stdout, "out\nout\nerr\n"); assert_eq!(*streamed.lock().unwrap(), "out\nout\nerr\n"); } + +/// `exec >log 2>&1` redirects the shell's own output: nothing reaches the +/// streaming callback until fd 1 is restored, and `>&3` to the saved +/// original stdout still does. +#[tokio::test] +async fn exec_redirected_output_is_not_streamed() { + let streamed = Arc::new(Mutex::new(String::new())); + let sink = streamed.clone(); + let mut bash = Bash::new(); + let result = bash + .exec_streaming( + "exec 3>&1 >/tmp/x.log 2>&1\necho one\nfor i in 1 2; do echo $i; done\necho saved >&3\nexec 1>&3 3>&-\necho back\ncat /tmp/x.log", + Box::new(move |stdout, stderr| { + let mut s = sink.lock().unwrap(); + s.push_str(&stdout.to_string()); + s.push_str(&stderr.to_string()); + }), + ) + .await + .unwrap(); + assert_eq!(result.stdout, "saved\nback\none\n1\n2\n"); + assert_eq!(*streamed.lock().unwrap(), "saved\nback\none\n1\n2\n"); +} diff --git a/crates/bashkit/tests/spec_cases/bash/exec-redirect-shell-output.test.sh b/crates/bashkit/tests/spec_cases/bash/exec-redirect-shell-output.test.sh new file mode 100644 index 000000000..55a01d7db --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/exec-redirect-shell-output.test.sh @@ -0,0 +1,62 @@ +### exec_stdout_to_file_then_restore +exec 3>&1 > /tmp/ex.log 2>&1 +echo one +ls /nonexistent_zz +echo two >&3 +exec 1>&3 3>&- +echo back +cat /tmp/ex.log +### expect +two +back +one +ls: cannot access '/nonexistent_zz': No such file or directory +### end + +### exec_inside_function_persists +exec 4>&1 +main() { exec > /tmp/m.log; echo inside; } +main +echo after +exec >&4 4>&- +echo restored +cat /tmp/m.log +### expect +restored +inside +after +### end + +### exec_redirect_spares_command_substitution +exec 3>&1 > /tmp/c.log +x=$(echo captured; ls /nonexistent_zz 2>&1 | wc -l) +echo "x=$x" | tr '\n' ' ' >&3 +echo +exec >&3 +echo "log=[$(cat /tmp/c.log)]" +### expect +x=captured 1 log=[] +### end + +### exec_stderr_to_dev_null +exec 2>/dev/null +ls /nonexistent_zz +echo "rc=$?" +### expect +rc=2 +### end + +### exec_append_and_pipeline +echo first > /tmp/a.log +exec 3>&1 >> /tmp/a.log +printf 'b\na\n' | sort +for i in 1 2; do echo "loop $i"; done +exec >&3 +cat /tmp/a.log +### expect +first +a +b +loop 1 +loop 2 +### end diff --git a/knowledge/log.md b/knowledge/log.md index dc75f08a2..52e7b2c63 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -17,6 +17,7 @@ * **Perf**: The CPython snapshot ships as xz (4.99 MB, was ~6.7 MB as gzip; decoded by `lzma-rs` in build.rs). The room went to preloading `urllib.request` (http.client, email), `tomllib`, `configparser` and `xml.etree.ElementTree`: their imports went from 0.1-0.47 s per call to ~0; `print(1)` unchanged at ~4.1 ms, first call ~20 ms. `doctest` works again (a `pdb` stand-in; `breakpoint()` prints a notice), `mailbox` dropped. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#startup-is-the-product-constraint). * **Feature**: `import requests`, `import httpx` and `import httpx2` work in CPython. They are bashkit's own compact implementations of the common API over the HTTP bridge, preloaded in the snapshot (import cost ~0 vs ~3.1 s for vendored upstream requests). http.client read timeouts now surface from `getresponse()`. L-CPY-003 updated, L-CPY-010 added. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#requests-and-httpx-bashkits-own-modules). * **Fix**: A backslash outside quotes keeps `$` and `` ` `` literal. `echo \$HOME` printed the home directory; it now prints `$HOME`, as do `a\$x`, `\${x}` and `x=\$y` (the lexer emits the same NUL literal marker as quoted `"\$"`). Found while working gap telemetry. +* **Fix**: `exec >log 2>&1` (and `exec 2>/dev/null`, `exec >>log`) redirects the script's own output. It used to be recorded in the fd table but only consulted by `>&N`, so everything still printed. fd 1/2 entries now route each top-level command's output (`route_exec_output`) and pause streaming for those streams. `exec 3>&1 >log; echo x >&3` reaches the caller through a passthrough, `exec 1>&3` restores, and `$( )`/pipelines keep their own stdout (`$(cmd 2>&1)` still captures). A failed `exec >/bad/path` leaves fd 1 alone with status 1. L-PROC-001 updated. * **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing). * **Fix**: Special parameters. `$0` inside a function or sourced file is the script/shell name (it was the function name). `$-` lists flags in bash order and includes the always-on `h` and `B`. `$BASH_SUBSHELL` counts `( )`, `$( )`, compound pipeline stages and jobs (simple-command stages stay at the parent level, as in bash). `${-…}`, `${?…}` and `${$…}` take operators, `${#}` is `$#`, and `${@:off:len}`/`${*:off:len}` slice the positional parameters (offset 0 is `$0`, negative counts from the end). `read -a` keeps empty fields (`IFS=, read -ra p <<< a,,b`), a single trailing delimiter does not add an empty field, and the last variable drops it when only one field is left. From the pseudo-linus oracle list. * **Feature**: `/proc/self` and `/proc/1` in the rootfs (static `status`, `comm`, `cmdline`, `stat`, `cgroup`, `mounts`, `exe` -> `/bin/bash`, `fd/0..2`), plus `/proc/uptime` and `/proc/mounts`. `$BASHPID` is set (1, same as `$$`; it was empty). Never host data (TM-INF-003). Step 3 of the level-up roadmap. Spec: `proc-self.test.sh`. diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index ce0172649..1782099f8 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -42,7 +42,7 @@ execution model. Evidence is a threat-model ID, a test, or `stance` | ID | Limitation | Why | Evidence | |----|------------|-----|----------| -| L-PROC-001 | `exec` does not replace the process; `exec cmd` runs cmd then stops execution (fd redirects work) | True process replace would break sandbox containment | TM-ESC-005 | +| L-PROC-001 | `exec` does not replace the process; `exec cmd` runs cmd then stops execution. fd redirects work; `exec >log 2>&1` routes the shell's output per top-level command, so output written earlier in the same top-level command (not streamed yet) also lands in the log, and output under the redirect reaches a streaming caller only when that command ends. `exec {var}>file` (allocate an fd into `var`) is not implemented | True process replace would break sandbox containment; there is no real fd 1 to swap, the shell's output is a value returned to the caller | TM-ESC-005, `exec-redirect-shell-output.test.sh` | | L-PROC-002 | Background jobs run concurrently (`jobs`, `wait -n`, `kill`, `ps`, `pgrep`), but cannot be stopped or resumed: `kill -STOP/-CONT` are ignored, there is no `suspend`/Ctrl-Z, `bg` is a no-op and `fg` just waits; jobs end when `exec()` returns | No terminal process groups in a virtual shell; a job must not outlive the call that owns it (TM-DOS-122) | `l_proc_002_no_job_control` | | L-PROC-003 | No process spawning; external commands run as builtins | Core sandbox model: no fork/exec escape surface | `l_proc_003_no_process_spawning` | | L-PROC-004 | `bash`/`sh` child shells nest at most 8 deep (and count against `max_function_depth`); deeper nesting fails `maximum nesting depth exceeded` | Child shells run in-process on one stack; unbounded nesting crashed the process (TM-DOS-125) | `l_proc_004_child_shell_depth` |