From c790733b3fbb357343fbf16fe68c6377fd12472e Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 14:41:35 +0000 Subject: [PATCH] feat(builtins): add fmt on the vendored uutils engine fmt fills and joins paragraphs with the uutils coreutils fmt line breaker (MIT, vendored linebreak.rs/parasplit.rs) and bashkit-owned option parsing and I/O. GNU option set plus uutils extras; GNU width rules and messages. Vendored panic!/unwrap paths became early returns. Line breaks can differ from GNU fmt (L-FMT-001): GNU fmt.c is GPL and is not ported. Claude-Session: https://claude.ai/code/session_019aFikmptPc91Fj4N2iDXQA --- NOTICE | 9 +- THIRD_PARTY_LICENSES/README.md | 9 +- crates/bashkit/src/builtins/fmt/linebreak.rs | 547 +++++++++++++ crates/bashkit/src/builtins/fmt/mod.rs | 350 +++++++++ crates/bashkit/src/builtins/fmt/parasplit.rs | 739 ++++++++++++++++++ crates/bashkit/src/builtins/mod.rs | 3 + crates/bashkit/src/interpreter/mod.rs | 1 + .../bashkit/tests/spec_cases/bash/fmt.test.sh | 124 +++ knowledge/foundations/builtins.md | 13 + knowledge/log.md | 1 + knowledge/operations/limitations.md | 1 + knowledge/status/builtins.json | 6 +- 12 files changed, 1798 insertions(+), 5 deletions(-) create mode 100644 crates/bashkit/src/builtins/fmt/linebreak.rs create mode 100644 crates/bashkit/src/builtins/fmt/mod.rs create mode 100644 crates/bashkit/src/builtins/fmt/parasplit.rs create mode 100644 crates/bashkit/tests/spec_cases/bash/fmt.test.sh diff --git a/NOTICE b/NOTICE index 92c390b33..b09e672c7 100644 --- a/NOTICE +++ b/NOTICE @@ -74,6 +74,11 @@ jaq-json (MIT), vendored in crates/bashkit/src/builtins/jq/jaq_json/ - Copyright (c) Michael Färber - Repository: https://github.com/01mf02/jaq +uutils/coreutils fmt (MIT), vendored in crates/bashkit/src/builtins/fmt/ + - linebreak.rs and parasplit.rs: fmt line breaking and paragraph splitting + - Copyright (c) uutils developers + - Repository: https://github.com/uutils/coreutils + grep, grep-regex, grep-searcher (MIT/Apache-2.0) - Text search functionality - Part of the ripgrep project by Andrew Gallant @@ -107,4 +112,6 @@ It is not affiliated with, endorsed by, or derived from: The shell syntax parser, interpreter, virtual filesystem, and all builtins (except where noted) are original implementations written in Rust. The jq value type in crates/bashkit/src/builtins/jq/jaq_json/ is a modified copy of -jaq-json (MIT, Michael Färber), see RUST DEPENDENCIES above. +jaq-json (MIT, Michael Färber), see RUST DEPENDENCIES above. The fmt line +breaker in crates/bashkit/src/builtins/fmt/ is a modified copy of uutils +coreutils fmt (MIT, uutils developers). diff --git a/THIRD_PARTY_LICENSES/README.md b/THIRD_PARTY_LICENSES/README.md index f67432975..94ec3c991 100644 --- a/THIRD_PARTY_LICENSES/README.md +++ b/THIRD_PARTY_LICENSES/README.md @@ -6,10 +6,13 @@ Bashkit's design or whose test case formats have inspired our testing approach. ## Important Notes 1. **Bashkit is an independent implementation.** No source code has been copied - from any of these projects, with one exception: the jq value type in + from any of these projects, with two exceptions: the jq value type in `crates/bashkit/src/builtins/jq/jaq_json/` is a modified copy of the jaq-json crate (MIT, Michael Färber, https://github.com/01mf02/jaq), kept - under its MIT license (`MIT.txt`). + under its MIT license (`MIT.txt`), and the `fmt` line breaker in + `crates/bashkit/src/builtins/fmt/` (`linebreak.rs`, `parasplit.rs`) is a + modified copy of uutils coreutils `fmt` (MIT, uutils developers, + https://github.com/uutils/coreutils), also under `MIT.txt`. 2. **Test cases are original.** While our testing methodology was inspired by projects like Oils, the actual test cases are written specifically for @@ -24,7 +27,7 @@ Bashkit's design or whose test case formats have inspired our testing approach. | File | Projects | License Type | |------|----------|--------------| | `APACHE-2.0.txt` | just-bash, Oils | Apache License 2.0 | -| `MIT.txt` | jq, jaq crates, most dependencies | MIT License | +| `MIT.txt` | jq, jaq crates, uutils coreutils, most dependencies | MIT License | | `LUCENT.txt` | One True AWK | Lucent Public License | ## Full Dependency Licenses diff --git a/crates/bashkit/src/builtins/fmt/linebreak.rs b/crates/bashkit/src/builtins/fmt/linebreak.rs new file mode 100644 index 000000000..5a603c68f --- /dev/null +++ b/crates/bashkit/src/builtins/fmt/linebreak.rs @@ -0,0 +1,547 @@ +// Vendored from uutils/coreutils (MIT, Copyright (c) uutils developers), +// src/uu/fmt/src/linebreak.rs at rev 6579cc2. Changes: output goes to a +// `Vec`, input is any `BufRead`, options come from `super::FmtOptions`. +// Keep in sync with upstream; see knowledge/foundations/builtins.md "fmt". + +// spell-checker:ignore (ToDO) INFTY MULT accum breakwords linebreak linebreaking linebreaks linelen maxlength minlength nchars ostream overlen parasplit plass posn powf punct signum slen sstart tabwidth tlen underlen winfo wlen wordlen + +use std::io::Write; +use std::mem; + +use super::FmtOptions; +use super::parasplit::{ParaWords, Paragraph, WordInfo}; + +struct BreakArgs<'a> { + opts: &'a FmtOptions, + init_len: usize, + indent: &'a [u8], + indent_len: usize, + uniform: bool, + ostream: &'a mut Vec, +} + +impl BreakArgs<'_> { + fn compute_width(&self, winfo: &WordInfo, posn: usize, fresh: bool) -> usize { + if fresh { + 0 + } else { + let post = winfo.after_tab; + match winfo.before_tab { + None => post, + Some(pre) => { + post + ((pre + posn) / self.opts.tabwidth + 1) * self.opts.tabwidth - posn + } + } + } + } +} + +pub fn break_lines( + para: &Paragraph, + opts: &FmtOptions, + ostream: &mut Vec, +) -> std::io::Result<()> { + // indent + let p_indent = ¶.indent_str; + let p_indent_len = para.indent_len; + + // words + let p_words = ParaWords::new(opts, para); + let mut p_words_words = p_words.words(); + + // the first word will *always* appear on the first line + // make sure of this here + let Some(winfo) = p_words_words.next() else { + return ostream.write_all(b"\n"); + }; + + // print the init, if it exists, and get its length + let p_init_len = winfo.word_nchars + + if opts.crown || opts.tagged { + // handle "init" portion + ostream.write_all(¶.init_str)?; + para.init_len + } else if !para.mail_header { + // for non-(crown, tagged) that's the same as a normal indent + ostream.write_all(p_indent)?; + p_indent_len + } else { + // except that mail headers get no indent at all + 0 + }; + + // write first word after writing init + ostream.write_all(winfo.word)?; + + // does this paragraph require uniform spacing? + let uniform = para.mail_header || opts.uniform; + + let mut break_args = BreakArgs { + opts, + init_len: p_init_len, + indent: p_indent, + indent_len: p_indent_len, + uniform, + ostream, + }; + + if opts.quick || para.mail_header { + break_simple(p_words_words, &mut break_args) + } else { + break_knuth_plass(p_words_words, &mut break_args) + } +} + +/// `break_simple` implements a "greedy" breaking algorithm: print words until +/// maxlength would be exceeded, then print a linebreak and indent and continue. +fn break_simple<'a, T: Iterator>>( + mut iter: T, + args: &mut BreakArgs<'a>, +) -> std::io::Result<()> { + iter.try_fold((args.init_len, false), |(l, prev_punct), winfo| { + accum_words_simple(args, l, prev_punct, winfo) + })?; + args.ostream.write_all(b"\n") +} + +fn accum_words_simple<'a>( + args: &mut BreakArgs<'a>, + l: usize, + prev_punct: bool, + winfo: &'a WordInfo<'a>, +) -> std::io::Result<(usize, bool)> { + // compute the length of this word, considering how tabs will expand at this position on the line + let wlen = winfo.word_nchars + args.compute_width(winfo, l, false); + + let slen = compute_slen( + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + + if l + wlen + slen > args.opts.width { + write_newline(args.indent, args.ostream)?; + write_with_spaces(&winfo.word[winfo.word_start..], 0, args.ostream)?; + Ok((args.indent_len + winfo.word_nchars, winfo.ends_punct)) + } else { + write_with_spaces(winfo.word, slen, args.ostream)?; + Ok((l + wlen + slen, winfo.ends_punct)) + } +} + +/// `break_knuth_plass` implements an "optimal" breaking algorithm in the style of +/// Knuth, D.E., and Plass, M.F. "Breaking Paragraphs into Lines." in Software, +/// Practice and Experience. Vol. 11, No. 11, November 1981. +/// +fn break_knuth_plass<'a, T: Clone + Iterator>>( + mut iter: T, + args: &mut BreakArgs<'a>, +) -> std::io::Result<()> { + // run the algorithm to get the breakpoints + let breakpoints = find_kp_breakpoints(iter.clone(), args); + + // iterate through the breakpoints (note that breakpoints is in reverse break order, so we .rev() it + let result: std::io::Result<(bool, bool)> = breakpoints.iter().rev().try_fold( + (false, false), + |(mut prev_punct, mut fresh), &(next_break, break_before)| { + if fresh { + write_newline(args.indent, args.ostream)?; + } + // at each breakpoint, keep emitting words until we find the word matching this breakpoint + for winfo in &mut iter { + let (slen, word) = slice_if_fresh( + fresh, + winfo.word, + winfo.word_start, + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + fresh = false; + prev_punct = winfo.ends_punct; + + // We find identical breakpoints here by comparing addresses of the references. + // This is OK because the backing vector is not mutating once we are linebreaking. + if std::ptr::eq(winfo, next_break) { + // OK, we found the matching word + if break_before { + write_newline(args.indent, args.ostream)?; + write_with_spaces(&winfo.word[winfo.word_start..], 0, args.ostream)?; + } else { + // breaking after this word, so that means "fresh" is true for the next iteration + write_with_spaces(word, slen, args.ostream)?; + fresh = true; + } + break; + } + write_with_spaces(word, slen, args.ostream)?; + } + Ok((prev_punct, fresh)) + }, + ); + let (mut prev_punct, mut fresh) = result?; + + // after the last linebreak, write out the rest of the final line. + for winfo in iter { + if fresh { + write_newline(args.indent, args.ostream)?; + } + let (slen, word) = slice_if_fresh( + fresh, + winfo.word, + winfo.word_start, + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + prev_punct = winfo.ends_punct; + fresh = false; + write_with_spaces(word, slen, args.ostream)?; + } + args.ostream.write_all(b"\n") +} + +struct LineBreak<'a> { + prev: usize, + linebreak: Option<&'a WordInfo<'a>>, + break_before: bool, + demerits: i64, + prev_rat: f32, + length: usize, + fresh: bool, +} + +#[allow(clippy::cognitive_complexity)] +fn find_kp_breakpoints<'a, T: Clone + Iterator>>( + iter: T, + args: &BreakArgs<'a>, +) -> Vec<(&'a WordInfo<'a>, bool)> { + let mut iter = iter.peekable(); + // set up the initial null linebreak + let mut linebreaks = vec![LineBreak { + prev: 0, + linebreak: None, + break_before: false, + demerits: 0, + prev_rat: f32::NAN, + length: args.init_len, + fresh: false, + }]; + // this vec holds the current active linebreaks; next_ holds the breaks that will be active for + // the next word + let mut active_breaks = vec![0]; + let mut next_active_breaks = vec![]; + + let stretch = args.opts.width - args.opts.goal; + let minlength = if args.opts.goal <= 10 { + 1 + } else { + args.opts.goal.max(stretch + 1) - stretch + }; + let mut new_linebreaks = vec![]; + let mut is_sentence_start = false; + while let Some(w) = iter.next() { + let next_word_sentence_final = is_next_word_sentence_final(iter.clone()); + // if this is the last word, we don't add additional demerits for this break + let (is_last_word, is_sentence_end) = match iter.peek() { + None => (true, true), + Some(&&WordInfo { + sentence_start: st, + new_line: nl, + .. + }) => (false, st || (nl && w.ends_punct)), + }; + + // should we be adding extra space at the beginning of the next sentence? + let slen = compute_slen(args.uniform, w.new_line, is_sentence_start, false); + + let mut best_active_demerits = i64::MAX; + let mut ld_idx = 0; + new_linebreaks.clear(); + let mut best_break_before: Option> = None; + let mut best_break_after: Option> = None; + next_active_breaks.clear(); + // go through each active break, extending it and possibly adding a new active + // break if we are above the minimum required length + #[allow(clippy::explicit_iter_loop)] + for &i in active_breaks.iter() { + let active = &mut linebreaks[i]; + if active.demerits < best_active_demerits { + best_active_demerits = active.demerits; + ld_idx = i; + } + + // Also consider a break before this word, so the previous line can end at the + // prior word when that yields a better global layout. + if !active.fresh && active.length >= minlength { + let (mut new_demerits, new_ratio) = compute_demerits( + args.opts.goal as isize - active.length as isize, + stretch, + w.word_nchars, + active.prev_rat, + ); + if is_sentence_end { + new_demerits = new_demerits.saturating_add(ORPHAN_BREAK_PENALTY); + } + let total_demerits = active.demerits.saturating_add(new_demerits); + if best_break_before + .as_ref() + .is_none_or(|best| total_demerits < best.demerits) + { + best_break_before = Some(LineBreak { + prev: i, + linebreak: Some(w), + break_before: true, + demerits: total_demerits, + prev_rat: new_ratio, + length: args.indent_len + w.word_nchars, + fresh: false, + }); + } + } + + // get the new length + let tlen = w.word_nchars + + args.compute_width(w, active.length, active.fresh) + + slen + + active.length; + + // if tlen is longer than args.opts.width, we drop this break from the active list + // otherwise, we extend the break, and possibly add a new break at this point + if tlen <= args.opts.width { + // this break will still be active next time + next_active_breaks.push(i); + // we can put this word on this line + active.fresh = false; + active.length = tlen; + + // if we're above the minlength, we can also consider breaking here + if tlen >= minlength { + let (mut new_demerits, new_ratio) = if is_last_word { + // there is no penalty for the final line's length + (0, 0.0) + } else { + compute_demerits( + args.opts.goal as isize - tlen as isize, + stretch, + w.word_nchars, + active.prev_rat, + ) + }; + + if !is_last_word && next_word_sentence_final { + new_demerits = new_demerits.saturating_add(ORPHAN_BREAK_PENALTY); + } + + let total_demerits = active.demerits.saturating_add(new_demerits); + if best_break_after + .as_ref() + .is_none_or(|best| total_demerits < best.demerits) + { + best_break_after = Some(LineBreak { + prev: i, + linebreak: Some(w), + break_before: false, + demerits: total_demerits, + prev_rat: new_ratio, + length: args.indent_len, + fresh: true, + }); + } + } + } + } + + if let Some(lb) = best_break_before { + new_linebreaks.push(lb); + } + if let Some(lb) = best_break_after { + new_linebreaks.push(lb); + } + + for lb in new_linebreaks.drain(..) { + next_active_breaks.push(linebreaks.len()); + linebreaks.push(lb); + } + + if next_active_breaks.is_empty() { + // every potential linebreak is too long! choose the linebreak with the least demerits, ld_idx + let new_break = + restart_active_breaks(args, &linebreaks[ld_idx], ld_idx, w, slen, minlength); + next_active_breaks.push(linebreaks.len()); + linebreaks.push(new_break); + } + // swap in new list of active breaks + mem::swap(&mut active_breaks, &mut next_active_breaks); + // If this was the last word in a sentence, the next one must be the first in the next. + is_sentence_start = is_sentence_end; + } + + // return the best path + build_best_path(&linebreaks, &active_breaks) +} + +fn build_best_path<'a>(paths: &[LineBreak<'a>], active: &[usize]) -> Vec<(&'a WordInfo<'a>, bool)> { + // of the active paths, we select the one with the fewest demerits + active + .iter() + .min_by_key(|&&a| paths[a].demerits) + .map(|&(mut best_idx)| { + let mut breakwords = vec![]; + // now, chase the pointers back through the break list, recording + // the words at which we should break + loop { + let next_best = &paths[best_idx]; + match next_best.linebreak { + None => return breakwords, + Some(prev) => { + breakwords.push((prev, next_best.break_before)); + best_idx = next_best.prev; + } + } + } + }) + .unwrap_or_default() +} + +// badness = BAD_MULT * abs(r) ^ 3 +const BAD_MULT: f32 = 200.0; +// DR_MULT is multiplier for delta-R between lines +const DR_MULT: f32 = 600.0; +// DL_MULT is penalty multiplier for short words at end of line +const DL_MULT: f32 = 10.0; +// Penalize breaks that leave the first word on the next line as the sentence-final word. +const ORPHAN_BREAK_PENALTY: i64 = 250_000_000; + +fn is_word_sentence_final(current: &WordInfo, next: Option<&WordInfo>) -> bool { + match next { + None => true, + Some(next_word) => next_word.sentence_start || (next_word.new_line && current.ends_punct), + } +} + +fn is_next_word_sentence_final<'a, T: Iterator>>(mut iter: T) -> bool { + let Some(next_word) = iter.next() else { + return false; + }; + is_word_sentence_final(next_word, iter.next()) +} + +fn compute_demerits(delta_len: isize, stretch: usize, wlen: usize, prev_rat: f32) -> (i64, f32) { + // how much stretch are we using? + let ratio = if delta_len == 0 { + 0.0f32 + } else { + delta_len as f32 / stretch as f32 + }; + + // compute badness given the stretch ratio + let bad_linelen = (BAD_MULT * ratio.powi(3).abs()) as i64; + + // we penalize lines ending in really short words + let bad_wordlen = if wlen >= stretch { + 0 + } else { + (DL_MULT + * ((stretch - wlen) as f32 / (stretch - 1) as f32) + .powi(3) + .abs()) as i64 + }; + + // we penalize lines that have very different ratios from previous lines + let bad_delta_r = if prev_rat.is_nan() { + 0 + } else { + (DR_MULT * ((ratio - prev_rat) / 2.0).powi(3).abs()) as i64 + }; + + let demerits_base = 1_i64 + .saturating_add(bad_linelen) + .saturating_add(bad_wordlen) + .saturating_add(bad_delta_r); + let demerits = demerits_base.saturating_mul(demerits_base); + + (demerits, ratio) +} + +fn restart_active_breaks<'a>( + args: &BreakArgs<'a>, + active: &LineBreak<'a>, + act_idx: usize, + w: &'a WordInfo<'a>, + slen: usize, + min: usize, +) -> LineBreak<'a> { + let (break_before, line_length) = if active.fresh { + // never break before a word if that word would be the first on a line + (false, args.indent_len) + } else { + // choose the lesser evil: breaking too early, or breaking too late + let wlen = w.word_nchars + args.compute_width(w, active.length, active.fresh); + let underlen = min as isize - active.length as isize; + let overlen = (wlen + slen + active.length) as isize - args.opts.width as isize; + if overlen > underlen { + // break early, put this word on the next line + (true, args.indent_len + w.word_nchars) + } else { + (false, args.indent_len) + } + }; + + // restart the linebreak. This will be our only active path. + LineBreak { + prev: act_idx, + linebreak: Some(w), + break_before, + demerits: 0, // this is the only active break, so we can reset the demerit count + prev_rat: if break_before { 1.0 } else { -1.0 }, + length: line_length, + fresh: !break_before, + } +} + +/// Number of spaces to add before a word, based on mode, newline, sentence start. +fn compute_slen(uniform: bool, newline: bool, start: bool, punct: bool) -> usize { + if uniform || newline { + if start || (newline && punct) { 2 } else { 1 } + } else { + 0 + } +} + +/// If we're on a fresh line, `slen=0` and we slice off leading whitespace. +/// Otherwise, compute `slen` and leave whitespace alone. +fn slice_if_fresh( + fresh: bool, + word: &[u8], + start: usize, + uniform: bool, + newline: bool, + sstart: bool, + punct: bool, +) -> (usize, &[u8]) { + if fresh { + (0, &word[start..]) + } else { + (compute_slen(uniform, newline, sstart, punct), word) + } +} + +/// Write a newline and add the indent. +fn write_newline(indent: &[u8], ostream: &mut Vec) -> std::io::Result<()> { + ostream.write_all(b"\n")?; + ostream.write_all(indent) +} + +/// Write the word, along with slen spaces. +fn write_with_spaces(word: &[u8], slen: usize, ostream: &mut Vec) -> std::io::Result<()> { + if slen == 2 { + ostream.write_all(b" ")?; + } else if slen == 1 { + ostream.write_all(b" ")?; + } + ostream.write_all(word) +} diff --git a/crates/bashkit/src/builtins/fmt/mod.rs b/crates/bashkit/src/builtins/fmt/mod.rs new file mode 100644 index 000000000..489533369 --- /dev/null +++ b/crates/bashkit/src/builtins/fmt/mod.rs @@ -0,0 +1,350 @@ +//! fmt builtin - fill and join paragraphs +//! +//! Important decision: the line breaker and paragraph splitter are vendored +//! from uutils/coreutils `fmt` (MIT, `linebreak.rs`, `parasplit.rs`), a +//! Knuth-Plass optimal-fit filler, not a greedy one. Bashkit is MIT, so GNU +//! `fmt.c` (GPL) must not be ported; uutils already targets GNU-compatible +//! output. Only option parsing and I/O are bashkit's. Width and goal follow +//! GNU's documented rules: goal defaults to 93% of the width, `-g` alone +//! sets the width to goal + 10, width is capped at 2500. + +mod linebreak; +mod parasplit; + +use async_trait::async_trait; + +use super::{Builtin, BuiltinHelper, Context, resolve_path}; +use crate::error::Result; +use crate::interpreter::ExecResult; + +/// The fmt builtin. +/// +/// Usage: fmt [-WIDTH] [OPTION]... [FILE]... +pub struct Fmt; + +impl BuiltinHelper for Fmt { + const NAME: &'static str = "fmt"; +} + +const USAGE: &str = "Usage: fmt [-WIDTH] [OPTION]... [FILE]...\nReformat each paragraph in the FILE(s), writing to standard output.\nThe option -WIDTH is an abbreviated form of --width=DIGITS.\n\nWith no FILE, or when FILE is -, read standard input.\n\n -c, --crown-margin preserve indentation of first two lines\n -p, --prefix=STRING reformat only lines beginning with STRING,\n reattaching the prefix to reformatted lines\n -s, --split-only split long lines, but do not refill\n -t, --tagged-paragraph indentation of first line different from second\n -u, --uniform-spacing one space between words, two after sentences\n -w, --width=WIDTH maximum line width (default of 75 columns)\n -g, --goal=WIDTH goal width (default of 93% of width)\n --help display this help and exit\n --version output version information and exit\n"; + +const DEFAULT_WIDTH: usize = 75; +const MAX_WIDTH: usize = 2500; + +/// Input handed to the vendored paragraph splitter. +type FileOrStdReader = std::io::Cursor>; + +/// Options read by the vendored `linebreak`/`parasplit` code. +struct FmtOptions { + crown: bool, + tagged: bool, + mail: bool, + split_only: bool, + prefix: Option, + xprefix: bool, + anti_prefix: Option, + xanti_prefix: bool, + uniform: bool, + quick: bool, + width: usize, + goal: usize, + tabwidth: usize, +} + +fn err(msg: String) -> ExecResult { + ExecResult::err(format!("fmt: {msg}\n"), 1) +} + +#[allow(clippy::result_large_err)] +/// GNU wording: bad digits vs. a number past `max`. +fn parse_width(value: &str, max: usize) -> std::result::Result { + let digits = !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()); + match value.parse::() { + Ok(w) if w <= max => Ok(w), + Err(_) if digits => Err(err(format!( + "invalid width: '{value}': Value too large for defined data type" + ))), + _ if digits => Err(err(format!( + "invalid width: '{value}': Numerical result out of range" + ))), + _ => Err(err(format!("invalid width: '{value}'"))), + } +} + +#[allow(clippy::result_large_err)] +/// Parse arguments into options and file operands. +fn parse_args(args: &[String]) -> std::result::Result<(FmtOptions, Vec<&str>), ExecResult> { + let mut opts = FmtOptions { + crown: false, + tagged: false, + mail: false, + split_only: false, + prefix: None, + xprefix: false, + anti_prefix: None, + xanti_prefix: false, + uniform: false, + quick: false, + width: DEFAULT_WIDTH, + goal: 0, + tabwidth: 8, + }; + let mut width: Option = None; + let mut goal: Option = None; + let mut files = Vec::new(); + let mut i = 0; + // Obsolete `fmt -WIDTH`, first argument only. + if let Some(digits) = args.first().and_then(|a| a.strip_prefix('-')) + && digits.starts_with(|c: char| c.is_ascii_digit()) + { + width = Some(parse_width(digits, MAX_WIDTH)?); + i = 1; + } + let mut only_files = false; + while i < args.len() { + let a = args[i].as_str(); + i += 1; + if only_files || a == "-" || !a.starts_with('-') { + files.push(a); + continue; + } + if a == "--" { + only_files = true; + continue; + } + if let Some(long) = a.strip_prefix("--") { + let (name, inline) = match long.split_once('=') { + Some((n, v)) => (n, Some(v.to_string())), + None => (long, None), + }; + let takes_value = matches!( + name, + "width" | "goal" | "prefix" | "skip-prefix" | "tab-width" + ); + let value = if takes_value { + match inline { + Some(v) => Some(v), + None => { + let v = args.get(i).cloned().ok_or_else(|| { + err(format!("option '--{name}' requires an argument")) + })?; + i += 1; + Some(v) + } + } + } else { + None + }; + let value = value.unwrap_or_default(); + match name { + "crown-margin" => opts.crown = true, + "tagged-paragraph" => opts.tagged = true, + "preserve-headers" => opts.mail = true, + "split-only" => opts.split_only = true, + "uniform-spacing" => opts.uniform = true, + "exact-prefix" => opts.xprefix = true, + "exact-skip-prefix" => opts.xanti_prefix = true, + "quick" => opts.quick = true, + "width" => width = Some(parse_width(&value, MAX_WIDTH)?), + "goal" => goal = Some(value), + "prefix" => opts.prefix = Some(value), + "skip-prefix" => opts.anti_prefix = Some(value), + "tab-width" => opts.tabwidth = parse_tab_width(&value)?, + _ => return Err(super::invalid_option("fmt", a, 1)), + } + continue; + } + // Short option cluster; a value option takes the rest or the next arg. + let bytes = a.as_bytes(); + let mut j = 1; + while j < bytes.len() { + let f = bytes[j]; + match f { + b'c' => opts.crown = true, + b't' => opts.tagged = true, + b'm' => opts.mail = true, + b's' => opts.split_only = true, + b'u' => opts.uniform = true, + b'x' => opts.xprefix = true, + b'X' => opts.xanti_prefix = true, + b'q' => opts.quick = true, + b'w' | b'g' | b'p' | b'P' | b'T' => { + let value = if j + 1 < bytes.len() { + a[j + 1..].to_string() + } else { + let v = args.get(i).cloned().ok_or_else(|| { + err(format!("option requires an argument -- '{}'", f as char)) + })?; + i += 1; + v + }; + match f { + b'w' => width = Some(parse_width(&value, MAX_WIDTH)?), + b'g' => goal = Some(value), + b'p' => opts.prefix = Some(value), + b'P' => opts.anti_prefix = Some(value), + _ => opts.tabwidth = parse_tab_width(&value)?, + } + break; + } + _ => return Err(super::invalid_option("fmt", a, 1)), + } + j += 1; + } + } + + // Mode precedence as in uutils: -s wins over -c, -c over -t. + if opts.crown { + opts.tagged = false; + } + if opts.split_only { + opts.crown = false; + opts.tagged = false; + } + opts.width = width.unwrap_or(DEFAULT_WIDTH); + opts.goal = match goal { + Some(g) => { + let g = parse_width(&g, opts.width)?; + if width.is_none() { + opts.width = g + 10; + } + g + } + None => opts.width * 187 / 200, + }; + if files.is_empty() { + files.push("-"); + } + Ok((opts, files)) +} + +#[allow(clippy::result_large_err)] +fn parse_tab_width(value: &str) -> std::result::Result { + value + .parse::() + .map(|t| t.max(1)) + .map_err(|_| err(format!("invalid tab width: '{value}'"))) +} + +/// Format one input into `out` with the vendored engine. +fn format_input(opts: &FmtOptions, input: Vec, out: &mut Vec) { + let mut reader: FileOrStdReader = std::io::Cursor::new(input); + for para in parasplit::ParagraphStream::new(opts, &mut reader) { + match para { + Err(line) => { + out.extend_from_slice(&line); + out.push(b'\n'); + } + // Writes to a Vec never fail. + Ok(para) => { + let _ = linebreak::break_lines(¶, opts, out); + } + } + } +} + +#[async_trait] +impl Builtin for Fmt { + async fn execute(&self, ctx: Context<'_>) -> Result { + if let Some(r) = Self::check_help(ctx.args, USAGE, Some("fmt (bashkit) 0.1")) { + return Ok(r); + } + let (opts, files) = match parse_args(ctx.args) { + Ok(v) => v, + Err(e) => return Ok(e), + }; + let mut out = Vec::new(); + let mut stderr = String::new(); + let mut exit_code = 0; + for file in files { + let input: Vec = if file == "-" { + ctx.stdin_bytes().map(<[u8]>::to_vec).unwrap_or_default() + } else { + let path = resolve_path(ctx.cwd, file); + match ctx.fs.read_file(&path).await { + Ok(bytes) => bytes, + Err(e) => { + stderr.push_str(&format!( + "fmt: cannot open '{file}' for reading: {}\n", + crate::error::io_error_reason(&e) + )); + exit_code = 1; + continue; + } + } + }; + ctx.consume_budget_work(1 + (input.len() / 1024) as u64)?; + format_input(&opts, input, &mut out); + } + let mut result = ExecResult::with_code(crate::StreamData::from(out), exit_code); + result.stderr = stderr.into(); + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fmt(args: &[&str], input: &str) -> String { + let args: Vec = args.iter().map(|s| s.to_string()).collect(); + let Ok((opts, _)) = parse_args(&args) else { + panic!("valid args"); + }; + let mut out = Vec::new(); + format_input(&opts, input.as_bytes().to_vec(), &mut out); + String::from_utf8(out).unwrap() + } + + #[test] + fn joins_short_lines() { + assert_eq!(fmt(&[], "a b\nc d\n"), "a b c d\n"); + } + + #[test] + fn keeps_blank_line_paragraphs() { + assert_eq!(fmt(&[], "a\nb\n\nc\n"), "a b\n\nc\n"); + } + + #[test] + fn two_spaces_after_sentence_end_at_line_end() { + assert_eq!( + fmt(&[], "Hello world.\nNext line.\n"), + "Hello world. Next line.\n" + ); + } + + #[test] + fn long_word_stays_whole() { + let word = "x".repeat(100); + assert_eq!( + fmt(&["-w", "20"], &format!("{word} y\n")), + format!("{word}\ny\n") + ); + } + + #[test] + fn no_trailing_newline_input() { + assert_eq!(fmt(&[], "a b"), "a b\n"); + } + + #[test] + fn empty_input() { + assert_eq!(fmt(&[], ""), ""); + } + + #[test] + fn width_rules() { + let args = |a: &[&str]| a.iter().map(|s| s.to_string()).collect::>(); + let (o, _) = parse_args(&args(&[])).ok().unwrap(); + assert_eq!((o.width, o.goal), (75, 70)); + let (o, _) = parse_args(&args(&["-g", "30"])).ok().unwrap(); + assert_eq!((o.width, o.goal), (40, 30)); + let (o, _) = parse_args(&args(&["-50"])).ok().unwrap(); + assert_eq!((o.width, o.goal), (50, 46)); + assert!(parse_args(&args(&["-w", "3000"])).is_err()); + assert!(parse_args(&args(&["-w", "20", "-g", "30"])).is_err()); + assert!(parse_args(&args(&["-w", "abc"])).is_err()); + assert!(parse_args(&args(&["-z"])).is_err()); + } +} diff --git a/crates/bashkit/src/builtins/fmt/parasplit.rs b/crates/bashkit/src/builtins/fmt/parasplit.rs new file mode 100644 index 000000000..4208c8455 --- /dev/null +++ b/crates/bashkit/src/builtins/fmt/parasplit.rs @@ -0,0 +1,739 @@ +// Vendored from uutils/coreutils (MIT, Copyright (c) uutils developers), +// src/uu/fmt/src/parasplit.rs at rev 6579cc2. Changes: output goes to a +// `Vec`, input is any `BufRead`, options come from `super::FmtOptions`. +// Keep in sync with upstream; see knowledge/foundations/builtins.md "fmt". + +// spell-checker:ignore (ToDO) INFTY MULT PSKIP accum aftertab beforetab breakwords fmt's formatline linebreak linebreaking linebreaks linelen maxlength minlength nchars noformat noformatline ostream overlen parasplit plass pmatch poffset posn powf prefixindent punct signum slen sstart tabwidth tlen underlen winfo wlen wordlen wordsplits xanti xprefix + +use std::io::BufRead; +use std::iter::Peekable; +use std::slice::Iter; + +use super::FileOrStdReader; +use super::FmtOptions; + +/// Column contribution of a decoded character toward GNU `fmt` `-w`/`--width`. +/// +/// Current GNU `fmt` (through at least 9.11) is not multibyte-aware: it measures +/// line length in UTF-8 bytes (pointer differences in `fmt.c`), not Unicode +/// display columns. Match that so wrapping agrees on CJK and other multibyte text. +/// When GNU switches to display-column width, revisit this. +fn char_width(c: char) -> usize { + c.len_utf8() +} + +/// Return the UTF-8 sequence length implied by a leading byte, or `None` if invalid. +fn utf8_char_width(byte: u8) -> Option { + // UTF-8 leading-byte ranges per Unicode Standard, Ch. 3, Table 3-7 and RFC 3629. + // 00..7F => 1 byte; C2..DF => 2 bytes; E0..EF => 3 bytes; F0..F4 => 4 bytes. + // Disallowed bytes include C0..C1 and F5..FF. + const ASCII_MAX: u8 = 0x7F; + const TWO_BYTE_START: u8 = 0xC2; + const TWO_BYTE_END: u8 = 0xDF; + const THREE_BYTE_START: u8 = 0xE0; + const THREE_BYTE_END: u8 = 0xEF; + const FOUR_BYTE_START: u8 = 0xF0; + const FOUR_BYTE_END: u8 = 0xF4; // up to U+10FFFF + + if byte <= ASCII_MAX { + return Some(1); + } + if (TWO_BYTE_START..=TWO_BYTE_END).contains(&byte) { + return Some(2); + } + if (THREE_BYTE_START..=THREE_BYTE_END).contains(&byte) { + return Some(3); + } + if (FOUR_BYTE_START..=FOUR_BYTE_END).contains(&byte) { + return Some(4); + } + None +} + +/// Decode a UTF-8 character starting at `start`, returning the char and bytes consumed. +fn decode_char(bytes: &[u8], start: usize) -> (Option, usize) { + let Some(&first) = bytes.get(start) else { + return (None, 1); + }; + if first < 0x80 { + return (Some(first as char), 1); + } + + let Some(width) = utf8_char_width(first) else { + return (None, 1); + }; + + if start + width > bytes.len() { + return (None, 1); + } + + match std::str::from_utf8(&bytes[start..start + width]) { + Ok(s) => (s.chars().next(), width), + Err(_) => (None, 1), + } +} + +struct DecodedCharInfo { + ch: Option, + consumed: usize, + width: usize, + is_ascii: bool, +} + +fn decode_char_info(bytes: &[u8], start: usize) -> DecodedCharInfo { + let (ch, consumed) = decode_char(bytes, start); + let (width, is_ascii) = match ch { + Some(c) => (char_width(c), c.is_ascii()), + None => (1, false), + }; + DecodedCharInfo { + ch, + consumed, + width, + is_ascii, + } +} + +/// Compute GNU-compatible width for a UTF-8 byte slice (byte length). +/// Invalid UTF-8 bytes are treated as width 1 each via `decode_char_info`. +fn byte_width(bytes: &[u8]) -> usize { + let mut width = 0; + let mut idx = 0; + while idx < bytes.len() { + let info = decode_char_info(bytes, idx); + width += info.width; + idx += info.consumed; + } + width +} + +/// GNU fmt has a more restrictive definition of whitespace than Unicode. +/// It only considers ASCII whitespace characters (space, tab, newline, etc.) +/// and excludes many Unicode whitespace characters like non-breaking spaces. +fn is_fmt_whitespace(c: char) -> bool { + // Only ASCII whitespace characters are considered whitespace in GNU fmt + matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0B' | '\x0C') +} + +fn is_fmt_whitespace_byte(b: u8) -> bool { + matches!(b, b' ' | b'\t' | b'\n' | b'\r' | 0x0B | 0x0C) +} + +// lines with PSKIP, lacking PREFIX, or which are entirely blank are +// NoFormatLines; otherwise, they are FormatLines +pub enum Line { + FormatLine(FileLine), + NoFormatLine(Vec, bool), +} + +impl Line { + /// when we know that it's a [`Line::FormatLine`], as in the [`ParagraphStream`] iterator + // bashkit: Option instead of panic! (no panics in builtins). + fn get_formatline(self) -> Option { + match self { + Self::FormatLine(fl) => Some(fl), + Self::NoFormatLine(..) => None, + } + } + + /// when we know that it's a [`Line::NoFormatLine`], as in the [`ParagraphStream`] iterator + fn get_noformatline(self) -> Option<(Vec, bool)> { + match self { + Self::NoFormatLine(s, b) => Some((s, b)), + Self::FormatLine(..) => None, + } + } +} + +/// Each line's prefix has to be considered to know whether to merge it with +/// the next line or not +pub struct FileLine { + line: Vec, + /// The end of the indent, always the start of the text + indent_end: usize, + /// The end of the PREFIX's indent, that is, the spaces before the prefix + prefix_indent_end: usize, + /// Printed length of indent taking into account tabs (GNU: byte-oriented) + indent_len: usize, + /// PREFIX indent length taking into account tabs + prefix_len: usize, +} + +/// Iterator that produces a stream of Lines from a file +pub struct FileLines<'a> { + opts: &'a FmtOptions, + reader: &'a mut FileOrStdReader, +} + +impl FileLines<'_> { + fn new<'b>(opts: &'b FmtOptions, reader: &'b mut FileOrStdReader) -> FileLines<'b> { + FileLines { opts, reader } + } + + /// returns true if this line should be formatted + fn match_prefix(&self, line: &[u8]) -> (bool, usize) { + let Some(prefix) = &self.opts.prefix else { + return (true, 0); + }; + + FileLines::match_prefix_generic(prefix.as_bytes(), line, self.opts.xprefix) + } + + /// returns true if this line should be formatted + fn match_anti_prefix(&self, line: &[u8]) -> bool { + let Some(anti_prefix) = &self.opts.anti_prefix else { + return true; + }; + + match FileLines::match_prefix_generic(anti_prefix.as_bytes(), line, self.opts.xanti_prefix) + { + (true, _) => false, + (_, _) => true, + } + } + + fn match_prefix_generic(pfx: &[u8], line: &[u8], exact: bool) -> (bool, usize) { + if line.starts_with(pfx) { + return (true, 0); + } + + if !exact { + let mut i = 0; + while i < line.len() { + if line[i..].starts_with(pfx) { + return (true, i); + } else if !is_fmt_whitespace_byte(line[i]) { + break; + } + i += 1; + } + } + + (false, 0) + } + + fn compute_indent(&self, bytes: &[u8], prefix_end: usize) -> (usize, usize, usize) { + let mut prefix_len = 0; + let mut indent_len = 0; + let mut indent_end = bytes.len(); + let mut idx = 0; + while idx < bytes.len() { + if idx == prefix_end { + // we found the end of the prefix, so this is the printed length of the prefix here + prefix_len = indent_len; + } + + let byte = bytes[idx]; + if idx >= prefix_end && !is_fmt_whitespace_byte(byte) { + indent_end = idx; + break; + } + + if byte == b'\t' { + indent_len = (indent_len / self.opts.tabwidth + 1) * self.opts.tabwidth; + idx += 1; + continue; + } + + let info = decode_char_info(bytes, idx); + indent_len += info.width; + idx += info.consumed; + } + if indent_end == bytes.len() { + indent_end = idx; + } + (indent_end, prefix_len, indent_len) + } +} + +impl Iterator for FileLines<'_> { + type Item = Line; + + fn next(&mut self) -> Option { + let mut buf = Vec::new(); + self.reader + .read_until(b'\n', &mut buf) + .ok() + .filter(|n| *n > 0)?; + if buf.ends_with(b"\n") { + buf.pop(); + if buf.ends_with(b"\r") { + buf.pop(); + } + } + let n = buf; + + // if this line is entirely whitespace, + // emit a blank line + // Err(true) indicates that this was a linebreak, + // which is important to know when detecting mail headers + if n.iter().all(|&b| is_fmt_whitespace_byte(b)) { + return Some(Line::NoFormatLine(Vec::new(), true)); + } + + let (pmatch, poffset) = self.match_prefix(&n[..]); + + // if this line does not match the prefix, + // emit the line unprocessed and iterate again + if !pmatch { + return Some(Line::NoFormatLine(n, false)); + } + + // if the line matches the prefix, but is blank after, + // don't allow lines to be combined through it (that is, + // treat it like a blank line, except that since it's + // not truly blank we will not allow mail headers on the + // following line) + if n[poffset + self.opts.prefix.as_ref().map_or(0, String::len)..] + .iter() + .all(|&b| is_fmt_whitespace_byte(b)) + { + return Some(Line::NoFormatLine(n, false)); + } + + // skip if this line matches the anti_prefix + // (NOTE definition of match_anti_prefix is TRUE if we should process) + if !self.match_anti_prefix(&n[..]) { + return Some(Line::NoFormatLine(n, false)); + } + + // figure out the indent, prefix, and prefixindent ending points + let prefix_end = poffset + self.opts.prefix.as_ref().map_or(0, String::len); + let (indent_end, prefix_len, indent_len) = self.compute_indent(&n[..], prefix_end); + + Some(Line::FormatLine(FileLine { + line: n, + indent_end, + prefix_indent_end: poffset, + indent_len, + prefix_len, + })) + } +} + +/// A paragraph : a collection of [`FileLines`] that are to be formatted +/// plus info about the paragraph's indentation +/// +/// We retain the raw bytes from the [`FileLine`]; the other info +/// is only there to help us in deciding how to merge lines into Paragraphs +pub struct Paragraph { + /// the lines of the file + lines: Vec>, + /// string representing the init, that is, the first line's indent + pub init_str: Vec, + /// printable length of the init string considering TABWIDTH + pub init_len: usize, + /// byte location of end of init in first line buffer + init_end: usize, + /// string representing indent + pub indent_str: Vec, + /// length of above + pub indent_len: usize, + /// byte location of end of indent (in crown and tagged mode, only applies to 2nd line and onward) + indent_end: usize, + /// we need to know if this is a mail header because we do word splitting differently in that case + pub mail_header: bool, +} + +/// An iterator producing a stream of paragraphs from a stream of lines +/// given a set of options. +pub struct ParagraphStream<'a> { + lines: Peekable>, + next_mail: bool, + opts: &'a FmtOptions, +} + +impl ParagraphStream<'_> { + pub fn new<'b>(opts: &'b FmtOptions, reader: &'b mut FileOrStdReader) -> ParagraphStream<'b> { + let lines = FileLines::new(opts, reader).peekable(); + // at the beginning of the file, we might find mail headers + ParagraphStream { + lines, + next_mail: true, + opts, + } + } + + /// Detect RFC822 mail header + fn is_mail_header(line: &FileLine) -> bool { + // a mail header begins with either "From " (envelope sender line) + // or with a sequence of printable ASCII chars (33 to 126, inclusive, + // except colon) followed by a colon. + if line.indent_end > 0 { + false + } else { + let l_slice = &line.line[..]; + if l_slice.starts_with(b"From ") { + true + } else { + let Some(colon_posn) = l_slice.iter().position(|&b| b == b':') else { + return false; + }; + + // header field must be nonzero length + if colon_posn == 0 { + return false; + } + + l_slice[..colon_posn] + .iter() + .all(|&b| (33..=126).contains(&(b as usize)) && b != b':') + } + } + } +} + +impl Iterator for ParagraphStream<'_> { + type Item = Result>; + + #[allow(clippy::cognitive_complexity)] + fn next(&mut self) -> Option>> { + // return a NoFormatLine in an Err; it should immediately be output + let noformat = match self.lines.peek()? { + Line::FormatLine(_) => false, + Line::NoFormatLine(_, _) => true, + }; + + // found a NoFormatLine, immediately dump it out + if noformat { + let (s, nm) = self.lines.next()?.get_noformatline()?; + self.next_mail = nm; + return Some(Err(s)); + } + + // found a FormatLine, now build a paragraph + let mut init_str = Vec::new(); + let mut init_end = 0; + let mut init_len = 0; + let mut indent_str = Vec::new(); + let mut indent_end = 0; + let mut indent_len = 0; + let mut prefix_len = 0; + let mut prefix_indent_end = 0; + let mut p_lines = Vec::new(); + + let mut in_mail = false; + let mut second_done = false; // for when we use crown or tagged mode + while let Some(Line::FormatLine(fl)) = self.lines.peek() { + // peek ahead + // need to explicitly force fl out of scope before we can call self.lines.next() + if p_lines.is_empty() { + // first time through the loop, get things set up + // detect mail header + if self.opts.mail && self.next_mail && ParagraphStream::is_mail_header(fl) { + in_mail = true; + // there can't be any indent or prefixindent because otherwise is_mail_header + // would fail since there cannot be any whitespace before the colon in a + // valid header field + indent_str.extend_from_slice(b" "); + indent_len = 2; + } else { + if self.opts.crown || self.opts.tagged { + init_str.extend_from_slice(&fl.line[..fl.indent_end]); + init_len = fl.indent_len; + init_end = fl.indent_end; + } else { + second_done = true; + } + + // these will be overwritten in the 2nd line of crown or tagged mode, but + // we are not guaranteed to get to the 2nd line, e.g., if the next line + // is a NoFormatLine or None. Thus, we set sane defaults the 1st time around + indent_str.extend_from_slice(&fl.line[..fl.indent_end]); + indent_len = fl.indent_len; + indent_end = fl.indent_end; + + // save these to check for matching lines + prefix_len = fl.prefix_len; + prefix_indent_end = fl.prefix_indent_end; + + // in tagged mode, add 4 spaces of additional indenting by default + // (gnu fmt's behavior is different: it seems to find the closest column to + // indent_end that is divisible by 3. But honestly that behavior seems + // pretty arbitrary. + // Perhaps a better default would be 1 TABWIDTH? But ugh that's so big. + if self.opts.tagged { + indent_str.extend_from_slice(b" "); + indent_len += 4; + } + } + } else if in_mail { + // lines following mail headers must begin with spaces + if fl.indent_end == 0 || (self.opts.prefix.is_some() && fl.prefix_indent_end == 0) { + break; // this line does not begin with spaces + } + } else if !second_done { + // now we have enough info to handle crown margin and tagged mode + + // in both crown and tagged modes we require that prefix_len is the same + if prefix_len != fl.prefix_len || prefix_indent_end != fl.prefix_indent_end { + break; + } + + // in tagged mode, indent has to be *different* on following lines + if self.opts.tagged + && indent_len - 4 == fl.indent_len + && indent_end == fl.indent_end + { + break; + } + + // this is part of the same paragraph, get the indent info from this line + indent_str.clear(); + indent_str.extend_from_slice(&fl.line[..fl.indent_end]); + indent_len = fl.indent_len; + indent_end = fl.indent_end; + + second_done = true; + } else { + // detect mismatch + if indent_end != fl.indent_end + || prefix_indent_end != fl.prefix_indent_end + || indent_len != fl.indent_len + || prefix_len != fl.prefix_len + { + break; + } + } + + let Some(fl) = self.lines.next().and_then(Line::get_formatline) else { + break; + }; + p_lines.push(fl.line); + + // when we're in split-only mode, we never join lines, so stop here + if self.opts.split_only { + break; + } + } + + // if this was a mail header, then the next line can be detected as one. Otherwise, it cannot. + // NOTE next_mail is true at ParagraphStream instantiation, and is set to true after a blank + // NoFormatLine. + self.next_mail = in_mail; + + Some(Ok(Paragraph { + lines: p_lines, + init_str, + init_len, + init_end, + indent_str, + indent_len, + indent_end, + mail_header: in_mail, + })) + } +} + +pub struct ParaWords<'a> { + opts: &'a FmtOptions, + para: &'a Paragraph, + words: Vec>, +} + +impl<'a> ParaWords<'a> { + pub fn new(opts: &'a FmtOptions, para: &'a Paragraph) -> Self { + let mut pw = ParaWords { + opts, + para, + words: Vec::new(), + }; + pw.create_words(); + pw + } + + fn create_words(&mut self) { + if self.para.mail_header { + // no extra spacing for mail headers; always exactly 1 space + // safe to trim_start on every line of a mail header, since the + // first line is guaranteed not to have any spaces + self.words.extend( + self.para + .lines + .iter() + .flat_map(|x| { + x.split(|b| is_fmt_whitespace_byte(*b)) + .filter(|segment| !segment.is_empty()) + }) + .map(|x| WordInfo { + word: x, + word_start: 0, + word_nchars: byte_width(x), + before_tab: None, + after_tab: 0, + sentence_start: false, + ends_punct: false, + new_line: false, + }), + ); + } else { + // first line + self.words.extend(if self.opts.crown || self.opts.tagged { + // crown and tagged mode has the "init" in the first line, so slice from there + WordSplit::new(self.opts, &self.para.lines[0][self.para.init_end..]) + } else { + // otherwise we slice from the indent + WordSplit::new(self.opts, &self.para.lines[0][self.para.indent_end..]) + }); + + if self.para.lines.len() > 1 { + let indent_end = self.para.indent_end; + let opts = self.opts; + self.words.extend( + self.para + .lines + .iter() + .skip(1) + .flat_map(|x| WordSplit::new(opts, &x[indent_end..])), + ); + } + } + } + + pub fn words(&'a self) -> Iter<'a, WordInfo<'a>> { + self.words.iter() + } +} + +struct WordSplit<'a> { + opts: &'a FmtOptions, + bytes: &'a [u8], + length: usize, + position: usize, + prev_punct: bool, +} + +impl WordSplit<'_> { + fn analyze_tabs(&self, bytes: &[u8]) -> (Option, usize, Option) { + let mut beforetab = None; + let mut aftertab = 0; + let mut word_start = None; + for (idx, b) in bytes.iter().enumerate() { + if !is_fmt_whitespace_byte(*b) { + word_start = Some(idx); + break; + } else if *b == b'\t' { + if beforetab.is_none() { + beforetab = Some(aftertab); + aftertab = 0; + } else { + aftertab = (aftertab / self.opts.tabwidth + 1) * self.opts.tabwidth; + } + } else { + aftertab += 1; + } + } + (beforetab, aftertab, word_start) + } + + fn new<'b>(opts: &'b FmtOptions, bytes: &'b [u8]) -> WordSplit<'b> { + let start = bytes + .iter() + .position(|&b| !is_fmt_whitespace_byte(b)) + .unwrap_or(bytes.len()); + let trimmed = &bytes[start..]; + WordSplit { + opts, + bytes: trimmed, + length: trimmed.len(), + position: 0, + prev_punct: false, + } + } + + fn is_punctuation_byte(b: u8) -> bool { + matches!(b, b'!' | b'.' | b'?') + } + + fn scan_word_end(&self, word_start: usize) -> (usize, usize, Option) { + let mut word_nchars = 0; + let mut idx = word_start; + let mut last_ascii = None; + while idx < self.length { + let info = decode_char_info(self.bytes, idx); + let is_whitespace = info.is_ascii && info.ch.is_some_and(is_fmt_whitespace); + if is_whitespace { + break; + } + word_nchars += info.width; + if info.is_ascii { + last_ascii = info.ch.map(|c| c as u8); + } else { + last_ascii = None; + } + idx += info.consumed; + } + (idx, word_nchars, last_ascii) + } +} + +pub struct WordInfo<'a> { + pub word: &'a [u8], + pub word_start: usize, + pub word_nchars: usize, + pub before_tab: Option, + pub after_tab: usize, + pub sentence_start: bool, + pub ends_punct: bool, + pub new_line: bool, +} + +// returns (&str, is_start_of_sentence) +impl<'a> Iterator for WordSplit<'a> { + type Item = WordInfo<'a>; + + fn next(&mut self) -> Option> { + if self.position >= self.length { + return None; + } + + let old_position = self.position; + let new_line = old_position == 0; + + // find the start of the next word, and record if we find a tab character + let (before_tab, after_tab, word_start) = + if let (b, a, Some(s)) = self.analyze_tabs(&self.bytes[old_position..]) { + (b, a, s + old_position) + } else { + self.position = self.length; + return None; + }; + + // find the beginning of the next whitespace + // note that this preserves the invariant that self.position + // points to whitespace character OR end of string + let (next_position, word_nchars, last_ascii) = self.scan_word_end(word_start); + self.position = next_position; + + let word_start_relative = word_start - old_position; + // if the previous sentence was punctuation and this sentence has >2 whitespace or one tab, is a new sentence. + let is_start_of_sentence = + self.prev_punct && (before_tab.is_some() || word_start_relative > 1); + + // now record whether this word ends in punctuation + let ends_punct = last_ascii.is_some_and(WordSplit::is_punctuation_byte); + self.prev_punct = ends_punct; + + let (word, word_start_relative, before_tab, after_tab) = if self.opts.uniform { + (&self.bytes[word_start..self.position], 0, None, 0) + } else { + ( + &self.bytes[old_position..self.position], + word_start_relative, + before_tab, + after_tab, + ) + }; + + Some(WordInfo { + word, + word_start: word_start_relative, + word_nchars, + before_tab, + after_tab, + sentence_start: is_start_of_sentence, + ends_punct, + new_line, + }) + } +} diff --git a/crates/bashkit/src/builtins/mod.rs b/crates/bashkit/src/builtins/mod.rs index 31b470e41..493b8581a 100644 --- a/crates/bashkit/src/builtins/mod.rs +++ b/crates/bashkit/src/builtins/mod.rs @@ -62,6 +62,7 @@ mod fc; mod fileops; mod find; mod flow; +mod fmt; mod fold; mod generated; mod glob_cmd; @@ -196,6 +197,7 @@ pub use fc::Fc; pub use fileops::{Chmod, Chown, Cp, Ln, Mkdir, Mktemp, Mv, Rm, Touch}; pub use find::Find; pub use flow::{Break, Colon, Continue, Exit, False, Return, True}; +pub use fmt::Fmt; pub use fold::Fold; pub use glob_cmd::GlobCmd; pub use grep::Grep; @@ -2059,6 +2061,7 @@ mod tests { "column", "join", "split", + "fmt", "fold", "expand", "unexpand", diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 279ee296a..d8527843a 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -1656,6 +1656,7 @@ impl Interpreter { "strings" => Strings, "tac" => Tac, "rev" => Rev, + "fmt" => Fmt, "fold" => Fold, "expand" => Expand, "unexpand" => Unexpand, diff --git a/crates/bashkit/tests/spec_cases/bash/fmt.test.sh b/crates/bashkit/tests/spec_cases/bash/fmt.test.sh new file mode 100644 index 000000000..880c2ae45 --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/fmt.test.sh @@ -0,0 +1,124 @@ +# fmt: optimal paragraph filling (line breaker vendored from uutils fmt). +# Cases marked bash_diff break lines differently from GNU fmt. + +### fmt_joins_lines +printf 'one two\nthree four\nfive\n' | fmt +### expect +one two three four five +### end + +### fmt_width +### bash_diff: uutils line breaker picks different breaks than GNU fmt +printf 'The quick brown fox jumps over the lazy dog and keeps running far away.\n' | fmt -w 30 +### expect +The quick brown fox jumps over +the lazy dog and keeps running +far away. +### end + +### fmt_obsolete_width +### bash_diff: uutils line breaker picks different breaks than GNU fmt +printf 'alpha beta gamma delta epsilon zeta eta theta iota kappa\n' | fmt -20 +### expect +alpha beta gamma +delta epsilon zeta +eta theta iota kappa +### end + +### fmt_sentences +printf 'One. Two words here.\nThree.\n' | fmt +### expect +One. Two words here. Three. +### end + +### fmt_uniform +printf 'a b c. d\n' | fmt -u +### expect +a b c. d +### end + +### fmt_split_only +printf 'short\nalpha beta gamma delta epsilon zeta eta theta\n' | fmt -s -w 20 +### expect +short +alpha beta gamma +delta epsilon zeta +eta theta +### end + +### fmt_paragraphs_and_indent +printf 'a b\nc d\n\n indented x\n y z\n' | fmt +### expect +a b c d + + indented x y z +### end + +### fmt_crown +### bash_diff: uutils line breaker picks different breaks than GNU fmt +printf ' first line\n second line words here and more words to wrap\n' | fmt -c -w 30 +### expect + first line second line words + here and more words to + wrap +### end + +### fmt_tagged +### bash_diff: uutils line breaker picks different breaks than GNU fmt +printf 'tag: alpha beta gamma delta epsilon zeta eta theta\n' | fmt -t -w 25 +### expect +tag: alpha beta gamma + delta epsilon zeta + eta theta +### end + +### fmt_prefix +printf '> alpha beta\n> gamma delta\ncode line\n> epsilon\n' | fmt -p '> ' +### expect +> alpha beta gamma delta +code line +> epsilon +### end + +### fmt_goal +printf 'one two three four five six seven eight nine ten eleven twelve\n' | fmt -g 20 +### expect +one two three four +five six seven eight +nine ten eleven twelve +### end + +### fmt_file +printf 'x y\nz\n' > /tmp/fmt_in.txt +fmt -w 10 /tmp/fmt_in.txt +### expect +x y z +### end + +### fmt_bad_width +LC_ALL=C fmt -w abc &1; echo "rc=$?" +### expect +fmt: invalid width: 'abc' +rc=1 +### end + +### fmt_width_too_large +LC_ALL=C fmt -w 3000 &1; echo "rc=$?" +### expect +fmt: invalid width: '3000': Numerical result out of range +rc=1 +### end + +### fmt_goal_over_width +LC_ALL=C fmt -w 20 -g 30 &1; echo "rc=$?" +### expect +fmt: invalid width: '30': Numerical result out of range +rc=1 +### end + +### fmt_missing_file +LC_ALL=C fmt /tmp/fmt_nosuch_file 2>&1; echo "rc=$?" +### expect +fmt: cannot open '/tmp/fmt_nosuch_file' for reading: No such file or directory +rc=1 +### end diff --git a/knowledge/foundations/builtins.md b/knowledge/foundations/builtins.md index 463dc3bd8..cbe803410 100644 --- a/knowledge/foundations/builtins.md +++ b/knowledge/foundations/builtins.md @@ -384,6 +384,19 @@ Deliberate gaps are L-MAKE-001..003 in [Known Limitations](../operations/limitations.md); amplification caps are TM-DOS-126 in the [Threat Model](../security/threat-model.md). +### fmt + +`fmt` vendors the uutils/coreutils `fmt` engine (MIT): `builtins/fmt/ +linebreak.rs` (Knuth-Plass optimal fit) and `parasplit.rs` (paragraphs, +prefixes, crown/tagged indents), pinned to the same uutils revision as the +generated argument files. Bashkit owns only `fmt/mod.rs`: option parsing and +I/O. Options are GNU's (`-w`, `-g`, `-c`, `-t`, `-s`, `-u`, `-p`, obsolete +`-WIDTH`) plus the uutils extras (`-m`, `-P`, `-x`, `-X`, `-q`, `-T`). Width +rules and error messages follow GNU: goal is 93% of the width, `-g` alone sets +the width to goal + 10, width is capped at 2500, goal at the width. Vendored +`panic!`/`unwrap` paths were turned into early returns. GNU `fmt.c` is GPL and +must not be ported; line-break differences are L-FMT-001. + ### Network Builtins `curl`, `wget`, `http` require the `http_client` feature + URL allowlist. diff --git a/knowledge/log.md b/knowledge/log.md index 196d2b0cc..d0c114171 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -5,6 +5,7 @@ * **Feature**: `sudo` and `busybox`, the top missing commands in the gap replay (`sudo tee /etc/x <