diff --git a/Cargo.lock b/Cargo.lock index 44ad79c98..999bf17b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ dependencies = [ name = "bashkit-cpython-wasm" version = "0.18.2" dependencies = [ - "flate2", + "lzma-rs", "wasmtime", ] @@ -977,7 +977,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1250,6 +1250,21 @@ version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32c" version = "0.6.8" @@ -1706,7 +1721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2953,6 +2968,16 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mach2" version = "0.6.0" @@ -4774,7 +4799,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4832,7 +4857,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5574,10 +5599,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6664,7 +6689,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/bashkit-cpython-wasm/Cargo.toml b/crates/bashkit-cpython-wasm/Cargo.toml index feb385aaa..05c016222 100644 --- a/crates/bashkit-cpython-wasm/Cargo.toml +++ b/crates/bashkit-cpython-wasm/Cargo.toml @@ -41,5 +41,6 @@ native = [] [build-dependencies] wasmtime = { workspace = true, features = ["cranelift", "parallel-compilation"] } -flate2 = { workspace = true } +# Decodes artifacts/python.wasm.xz (pure Rust, build time only). +lzma-rs = "0.3" diff --git a/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md b/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md index e9ffefa80..ef1723bb9 100644 --- a/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md +++ b/crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md @@ -7,14 +7,14 @@ libraries statically linked into it (see `guest/build.sh` and | Component | Version | License | Where | |-----------|---------|---------|-------| -| CPython (interpreter and stdlib) | 3.14.8 | Python-2.0 (PSF License Agreement and predecessors) | `artifacts/python.wasm.gz`, `artifacts/python314.zip` | -| libmpdec (`decimal`) | bundled with CPython | BSD-2-Clause | `artifacts/python.wasm.gz` | -| Expat (`pyexpat`, `xml.etree`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` | -| HACL* (`hashlib` md5/sha1/sha2/sha3/blake2, `hmac`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` | -| zlib | 1.3.1 | Zlib | `artifacts/python.wasm.gz` | -| SQLite | 3.50.4 | Public domain (blessing) | `artifacts/python.wasm.gz` | -| wasi-libc (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT; musl parts MIT | `artifacts/python.wasm.gz` | -| compiler-rt builtins (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception | `artifacts/python.wasm.gz` | +| CPython (interpreter and stdlib) | 3.14.8 | Python-2.0 (PSF License Agreement and predecessors) | `artifacts/python.wasm.xz`, `artifacts/python314.zip` | +| libmpdec (`decimal`) | bundled with CPython | BSD-2-Clause | `artifacts/python.wasm.xz` | +| Expat (`pyexpat`, `xml.etree`) | bundled with CPython | MIT | `artifacts/python.wasm.xz` | +| HACL* (`hashlib` md5/sha1/sha2/sha3/blake2, `hmac`) | bundled with CPython | MIT | `artifacts/python.wasm.xz` | +| zlib | 1.3.1 | Zlib | `artifacts/python.wasm.xz` | +| SQLite | 3.50.4 | Public domain (blessing) | `artifacts/python.wasm.xz` | +| wasi-libc (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT; musl parts MIT | `artifacts/python.wasm.xz` | +| compiler-rt builtins (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception | `artifacts/python.wasm.xz` | The full license texts that require reproduction follow. diff --git a/crates/bashkit-cpython-wasm/artifacts/MANIFEST b/crates/bashkit-cpython-wasm/artifacts/MANIFEST index 6e6c86e5b..449af0507 100644 --- a/crates/bashkit-cpython-wasm/artifacts/MANIFEST +++ b/crates/bashkit-cpython-wasm/artifacts/MANIFEST @@ -3,5 +3,5 @@ wasi_sdk=24 wasmtime_wizer=49.0.1 zlib=1.3.1 sqlite=sqlite-amalgamation-3500400 -31c43d757812151582ff3e274f0d9c65cf210064a46646a5a73a68704b83329b python.wasm.gz -7c1fec908a3c250e0591a4892f3dfbaa73e5c81798dfa33b1dc80e18931dbb74 python314.zip +77058ae157db6f10e3fe9e73745147627a05bca03ee6a6c50a0a5f04ea1c7fa6 python.wasm.xz +bfe72099bf1ab706b9de9a8856f509ac5db4fb1c86c29a49c622d37490ce388e python314.zip diff --git a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz b/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz deleted file mode 100644 index 58fdb7823..000000000 Binary files a/crates/bashkit-cpython-wasm/artifacts/python.wasm.gz and /dev/null differ diff --git a/crates/bashkit-cpython-wasm/artifacts/python.wasm.xz b/crates/bashkit-cpython-wasm/artifacts/python.wasm.xz new file mode 100644 index 000000000..875554ba6 Binary files /dev/null and b/crates/bashkit-cpython-wasm/artifacts/python.wasm.xz differ diff --git a/crates/bashkit-cpython-wasm/artifacts/python314.zip b/crates/bashkit-cpython-wasm/artifacts/python314.zip index 81abd3bb9..bda9a819d 100644 Binary files a/crates/bashkit-cpython-wasm/artifacts/python314.zip and b/crates/bashkit-cpython-wasm/artifacts/python314.zip differ diff --git a/crates/bashkit-cpython-wasm/build.rs b/crates/bashkit-cpython-wasm/build.rs index e117231bd..e1b2d539b 100644 --- a/crates/bashkit-cpython-wasm/build.rs +++ b/crates/bashkit-cpython-wasm/build.rs @@ -5,7 +5,6 @@ //! product feature). Set `BASHKIT_CPYTHON_CWASM=/path/python.cwasm` to reuse a //! module precompiled by the same wasmtime version and configuration. -use std::io::Read; use std::path::PathBuf; include!("src/config.rs"); @@ -13,7 +12,7 @@ include!("src/config.rs"); fn main() { let manifest = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()); let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("python.cwasm"); - let snapshot = manifest.join("artifacts/python.wasm.gz"); + let snapshot = manifest.join("artifacts/python.wasm.xz"); println!("cargo:rerun-if-changed={}", snapshot.display()); println!("cargo:rerun-if-changed=src/config.rs"); println!("cargo:rerun-if-env-changed=BASHKIT_CPYTHON_CWASM"); @@ -23,9 +22,11 @@ fn main() { return; } + // xz, not gzip: ~30% smaller, which keeps the crate under crates.io's + // 10 MiB cap with more modules preloaded. Decoded here only. let mut wasm = Vec::new(); - flate2::read::GzDecoder::new(std::fs::File::open(&snapshot).expect("open snapshot")) - .read_to_end(&mut wasm) + let file = std::fs::File::open(&snapshot).expect("open snapshot"); + lzma_rs::xz_decompress(&mut std::io::BufReader::new(file), &mut wasm) .expect("decompress snapshot"); // The build script's own cfg is the host's; read the crate feature and // the cross-compilation target from Cargo instead. diff --git a/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py index 484b30ec6..70832f952 100644 --- a/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py +++ b/crates/bashkit-cpython-wasm/guest/_bashkit_boot.py @@ -47,6 +47,8 @@ # bashkit's own HTTP clients over the host bridge: preloaded so # `import requests` / `import httpx` cost nothing per call. "requests", "requests.utils", "httpx", "httpx2", + # Common in agent scripts and 0.1-0.5 s each to import on Pulley. + "urllib.request", "tomllib", "configparser", "xml.etree.ElementTree", ) _USAGE = """\ diff --git a/crates/bashkit-cpython-wasm/guest/build.sh b/crates/bashkit-cpython-wasm/guest/build.sh index ba1d9aab2..96688dca2 100755 --- a/crates/bashkit-cpython-wasm/guest/build.sh +++ b/crates/bashkit-cpython-wasm/guest/build.sh @@ -11,11 +11,12 @@ # - The stdlib ships as one deflate-compressed zip of .py sources. The # snapshot caches the zip directory, so the runtime must serve the exact # same zip bytes; both artifacts are produced together by this script. -# - Output is a gzip of the stripped snapshot plus the zip. The crate's +# - Output is an xz of the stripped snapshot plus the zip (xz, not gzip: +# ~30% smaller, which keeps the crate under the crates.io 10 MiB cap). The crate's # build.rs compiles the snapshot to Pulley bytecode ahead of time. # # Usage: guest/build.sh [WORK_DIR] -# Requires: curl, tar, unzip, python3 (>=3.11), make, a C compiler for the +# Requires: curl, tar, unzip, xz, python3 (>=3.11), make, a C compiler for the # native build Python. Everything else is downloaded into WORK_DIR. set -euo pipefail @@ -130,11 +131,13 @@ cp -r "$HERE/requests" "$WORK/stdlib/requests" cd "$WORK/stdlib" # Not usable or not useful in a sandboxed, single-threaded, headless guest. rm -rf test idlelib tkinter turtledemo ensurepip venv pydoc_data turtle.py \ - _pyrepl/__pycache__ curses dbm/gnu.py dbm/ndbm.py multiprocessing concurrent/futures/process.py + _pyrepl/__pycache__ curses mailbox.py dbm/gnu.py dbm/ndbm.py multiprocessing concurrent/futures/process.py # No FFI, TLS, sockets or TTY in the guest: these can never work. rm -rf ctypes ssl.py ftplib.py imaplib.py poplib.py smtplib.py socketserver.py \ http/server.py wsgiref xmlrpc webbrowser.py _pyrepl pdb.py bdb.py pydoc.py \ _aix_support.py _android_support.py _ios_support.py _osx_support.py + # Debugger stand-in: doctest imports pdb; breakpoint() prints a notice. + cp "$HERE/pdb.py" pdb.py # HTTP goes through the host's egress pipeline, not sockets # (_bashkit_http.py); http.client patches itself when first imported. printf '\n# bashkit: connections go through the host (see _bashkit_http).\nimport _bashkit_http\n_bashkit_http.patch_http_client(globals())\ndel _bashkit_http\n' >>http/client.py @@ -185,7 +188,8 @@ case "$out" in esac mkdir -p "$CRATE/artifacts" -gzip -9 -n -c "$WORK/python.wasm" >"$CRATE/artifacts/python.wasm.gz" +rm -f "$CRATE/artifacts/python.wasm.gz" +xz -9e -T1 -c "$WORK/python.wasm" >"$CRATE/artifacts/python.wasm.xz" cp "$ROOT/usr/local/lib/python314.zip" "$CRATE/artifacts/python314.zip" ( cd "$CRATE/artifacts" @@ -195,7 +199,7 @@ cp "$ROOT/usr/local/lib/python314.zip" "$CRATE/artifacts/python314.zip" echo "wasmtime_wizer=${WASMTIME_VERSION}" echo "zlib=${ZLIB_VERSION}" echo "sqlite=${SQLITE_AMALGAMATION}" - sha256sum python.wasm.gz python314.zip + sha256sum python.wasm.xz python314.zip } >MANIFEST ) ls -la "$CRATE/artifacts" diff --git a/crates/bashkit-cpython-wasm/guest/pdb.py b/crates/bashkit-cpython-wasm/guest/pdb.py new file mode 100644 index 000000000..f3c980073 --- /dev/null +++ b/crates/bashkit-cpython-wasm/guest/pdb.py @@ -0,0 +1,46 @@ +"""pdb stand-in for bashkit's CPython guest. + +There is no interactive terminal in the sandbox, so the real debugger is +not shipped. This stub keeps modules that import pdb (doctest, unittest +helpers) working: `breakpoint()` / `pdb.set_trace()` print a notice and +continue; `pdb.run*` raise RuntimeError. +""" + +import sys + +_MSG = "pdb: debugger not available in the bashkit sandbox; continuing" + + +class Pdb: + """Constructible (doctest creates one per run); debugging is a no-op.""" + + def __init__(self, *args, **kwargs): + pass + + def set_trace(self, *args, **kwargs): + print(_MSG, file=sys.stderr) + + def reset(self): + pass + + def set_continue(self): + pass + + +def set_trace(*args, header=None, **kwargs): + print(_MSG, file=sys.stderr) + + +def post_mortem(*args, **kwargs): + print(_MSG, file=sys.stderr) + + +def pm(): + post_mortem() + + +def run(*args, **kwargs): + raise RuntimeError("pdb is not available in the bashkit sandbox") + + +runeval = runcall = runctx = run diff --git a/crates/bashkit/docs/cpython.md b/crates/bashkit/docs/cpython.md index 8043c7d23..a3f171b5f 100644 --- a/crates/bashkit/docs/cpython.md +++ b/crates/bashkit/docs/cpython.md @@ -203,7 +203,8 @@ with httpx.Client(base_url="https://api.example.com", headers={"X-Key": "..."}) `_hashlib` (OpenSSL), `tkinter`, `curses`, `readline`, `dbm.gnu` are not built. `hashlib` still provides md5, sha1, sha2, sha3 and blake2. - **No interactive mode**: `python3` with no program reads one from stdin; - there is no REPL, and `pdb` and `pydoc` (`help()`) are not shipped. + there is no REPL, `pydoc` (`help()`) is not shipped, and `breakpoint()` + prints a notice and continues (no debugger). - **Stdlib is bytecode only**: tracebacks through stdlib code show no source line, and `inspect.getsource()` fails on stdlib objects. Your own code keeps full tracebacks. Non-HTTP network clients and servers (`smtplib`, diff --git a/crates/bashkit/fuzz/Cargo.lock b/crates/bashkit/fuzz/Cargo.lock index 7b20e20ff..25aa40eb6 100644 --- a/crates/bashkit/fuzz/Cargo.lock +++ b/crates/bashkit/fuzz/Cargo.lock @@ -141,7 +141,7 @@ dependencies = [ name = "bashkit-cpython-wasm" version = "0.18.2" dependencies = [ - "flate2", + "lzma-rs", "wasmtime", ] @@ -241,6 +241,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -533,6 +539,21 @@ version = "0.135.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b73d7e686f3518831f88ffe88be02c6c92be09930fc5ab90a3dfd9a4de3815e" +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1217,6 +1238,16 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mach2" version = "0.6.0" diff --git a/crates/bashkit/tests/integration/cpython_http_tests.rs b/crates/bashkit/tests/integration/cpython_http_tests.rs index 48dbf83ec..fba67a246 100644 --- a/crates/bashkit/tests/integration/cpython_http_tests.rs +++ b/crates/bashkit/tests/integration/cpython_http_tests.rs @@ -743,7 +743,7 @@ except requests.ConnectionError as e: import sys before = set(sys.modules) import requests, httpx, httpx2 -heavy = {"http.client", "email", "urllib3", "ssl", "http.cookiejar"} +heavy = {"urllib3", "ssl", "http.cookiejar", "charset_normalizer"} print(sorted(set(sys.modules) - before), sorted(heavy & set(sys.modules))) "#, ) diff --git a/crates/bashkit/tests/integration/cpython_integration_tests.rs b/crates/bashkit/tests/integration/cpython_integration_tests.rs index e77c8b7ac..5377549c4 100644 --- a/crates/bashkit/tests/integration/cpython_integration_tests.rs +++ b/crates/bashkit/tests/integration/cpython_integration_tests.rs @@ -535,7 +535,7 @@ async fn stdlib_is_bytecode_only() { let out = ok("python3 -c ' import email.message, http.client, xml.dom.minidom, json print(json.__file__.endswith(\".pyc\"), email.message.__file__.endswith(\".pyc\")) -for m in (\"ctypes\", \"ssl\", \"smtplib\", \"pdb\", \"pydoc\", \"_pydecimal\"): +for m in (\"ctypes\", \"ssl\", \"smtplib\", \"bdb\", \"pydoc\", \"_pydecimal\", \"mailbox\"): try: __import__(m) print(m, \"present\") @@ -548,6 +548,34 @@ print(decimal.Decimal(\"1.10\") + 1, datetime.datetime.strptime(\"2020\", \"%Y\" assert_eq!(out, "True True\n2.10 2020\n"); } +/// The real debugger is not shipped; a stand-in keeps doctest importable and +/// makes `breakpoint()` print a notice and continue. +#[tokio::test] +async fn doctest_works_and_breakpoint_continues() { + let r = Bash::builder() + .cpython() + .build() + .exec( + "cat > /m.py <<'EOF' +def add(a, b): + \"\"\" + >>> add(1, 2) + 3 + \"\"\" + return a + b +EOF +cd / && python3 -c 'import doctest, m; print(doctest.testmod(m)); breakpoint(); print(\"after\")'", + ) + .await + .unwrap(); + assert_eq!( + r.stdout, "TestResults(failed=0, attempted=1)\nafter\n", + "{}", + r.stderr + ); + assert!(r.stderr.contains("debugger not available"), "{}", r.stderr); +} + #[tokio::test] async fn local_modules_are_reimported_each_call() { let out = ok( diff --git a/knowledge/log.md b/knowledge/log.md index a7a311d0a..49db29071 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,6 +4,7 @@ * **Fix**: `${x#pat}`/`${x%pat}` follow the glob grammar. A lone `*` matches empty for the short forms (`${x#*}` and `${x%*}` keep `x`; they dropped a character), and patterns with `?` or several `*` (`${x%?}`, `${p#*/*/}`, `${p##*.*.}`) now match instead of being compared as literal text. Single-`*` patterns keep their linear fast path. * **Fix**: Quoted or escaped glob characters are literal everywhere. `echo a\*`, `a"*"` and `a'?'` no longer glob; `case`/`[[ == ]]` patterns match `"$x"`, `'a*'`, `a\*` and `\[s\]` literally; `"*"zz*` with no match prints `*zz*` (escapes no longer leak); `"$x"zz*` no longer reads `$xzz`; `'a'$x` expands and `'a'*` globs; `a{1,2}"*"` still brace-expands. The lexer escapes quoted glob characters in mixed words (the existing `QuotedGlobWord` convention, now with quote markers); `expand_word` returns unescaped text for non-glob uses; patterns use `expand_pattern_word`. Known gap: data from an unquoted expansion inside such a word loses `\` before glob characters when nothing matches. +* **Perf**: The CPython snapshot ships as xz (4.99 MB, was ~6.7 MB as gzip; decoded by `lzma-rs` in build.rs). The room went to preloading `urllib.request` (http.client, email), `tomllib`, `configparser` and `xml.etree.ElementTree`: their imports went from 0.1-0.47 s per call to ~0; `print(1)` unchanged at ~4.1 ms, first call ~20 ms. `doctest` works again (a `pdb` stand-in; `breakpoint()` prints a notice), `mailbox` dropped. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#startup-is-the-product-constraint). * **Feature**: `import requests`, `import httpx` and `import httpx2` work in CPython. They are bashkit's own compact implementations of the common API over the HTTP bridge, preloaded in the snapshot (import cost ~0 vs ~3.1 s for vendored upstream requests). http.client read timeouts now surface from `getresponse()`. L-CPY-003 updated, L-CPY-010 added. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#requests-and-httpx-bashkits-own-modules). * **Fix**: A backslash outside quotes keeps `$` and `` ` `` literal. `echo \$HOME` printed the home directory; it now prints `$HOME`, as do `a\$x`, `\${x}` and `x=\$y` (the lexer emits the same NUL literal marker as quoted `"\$"`). Found while working gap telemetry. * **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing). diff --git a/knowledge/operations/dependencies.md b/knowledge/operations/dependencies.md index bb37605fd..f31bca772 100644 --- a/knowledge/operations/dependencies.md +++ b/knowledge/operations/dependencies.md @@ -160,6 +160,13 @@ worth keeping so the question does not get re-litigated from scratch. of `Quotable`), but they are imported by *generated* coreutils-port files, so removal means carrying rewrite rules in the porter. Low return. +## Build-only dependencies of `bashkit-cpython-wasm` + +- `lzma-rs` (pure Rust, MIT) decodes the xz-compressed CPython snapshot in + `build.rs`. Never linked into a binary. xz keeps the crate under the + crates.io 10 MiB cap with more modules preloaded (see + [CPython WebAssembly Runtime](../runtimes/cpython-wasm.md)). + ## Vendored crates - **`jaq-json`** is vendored into the jq builtin so its value operations can diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 9dc6fcac8..758eb2771 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -134,7 +134,7 @@ Boundaries of the WebAssembly CPython guest; see | L-CPY-007 | No interactive REPL; `python3` with no program reads one from stdin | No TTY inside the sandbox | stance | | L-CPY-008 | Guest memory per call is capped at 1 GiB even if `max_memory` is higher | Pooled instance slots have a fixed maximum size | [CPython WebAssembly Runtime](../runtimes/cpython-wasm.md) | | L-CPY-010 | `requests` and `httpx` are subsets: no retries, proxies, client certificates, HTTP/2, digest auth, streaming uploads or OPTIONS; bodies are buffered; `AsyncClient` requests run sequentially | Upstream packages cost ~3 s per call to import on Pulley; the subset is preloaded in the snapshot and costs nothing | `requests_lib`, `httpx_lib` in `cpython_http_tests` | -| L-CPY-009 | Stdlib ships as bytecode only: tracebacks show no source line for stdlib frames, `inspect.getsource()` fails on stdlib objects. Modules that cannot work in the guest are not shipped: `ctypes`, `ssl`, `ftplib`, `imaplib`, `poplib`, `smtplib`, `socketserver`, `http.server`, `wsgiref`, `xmlrpc`, `webbrowser`, `pdb`, `bdb`, `pydoc` (so `help()`), `_pyrepl` | Compiling source on Pulley costs seconds per import; sources would push the crate past the crates.io 10 MiB cap. No FFI, TLS, sockets or TTY exist | `stdlib_is_bytecode_only` (cpython_integration_tests) | +| L-CPY-009 | Stdlib ships as bytecode only: tracebacks show no source line for stdlib frames, `inspect.getsource()` fails on stdlib objects. Modules that cannot work in the guest are not shipped: `ctypes`, `ssl`, `ftplib`, `imaplib`, `poplib`, `smtplib`, `socketserver`, `http.server`, `wsgiref`, `xmlrpc`, `webbrowser`, `bdb`, `pydoc` (so `help()`), `_pyrepl`, `mailbox`; `pdb` is a stand-in (`breakpoint()` prints a notice and continues, doctest works) | Compiling source on Pulley costs seconds per import; sources would push the crate past the crates.io 10 MiB cap. No FFI, TLS, sockets or TTY exist | `stdlib_is_bytecode_only` (cpython_integration_tests) | ## Text Processing diff --git a/knowledge/operations/release-process.md b/knowledge/operations/release-process.md index 5b3c8739b..5c75357a5 100644 --- a/knowledge/operations/release-process.md +++ b/knowledge/operations/release-process.md @@ -82,7 +82,7 @@ silently failed. `cargo publish` packaging step, missing files, version drift): - `cargo publish --dry-run -p bashkit-cpython-wasm` must succeed and the `.crate` must stay under crates.io's 10 MiB limit (it embeds the CPython - snapshot; ~9.8 MiB after the requests/httpx modules, 236 KB under the cap). `bashkit` depends on the same version + snapshot as xz; ~8.2 MiB with requests/httpx and urllib.request preloaded). `bashkit` depends on the same version of it, so `publish.yml` publishes it first and the `bashkit` dry-run only resolves once that version is live. - `cargo publish --dry-run -p bashkit` must succeed. Package diff --git a/knowledge/runtimes/cpython-wasm.md b/knowledge/runtimes/cpython-wasm.md index 8d7403bb0..6f0d3fe61 100644 --- a/knowledge/runtimes/cpython-wasm.md +++ b/knowledge/runtimes/cpython-wasm.md @@ -71,7 +71,20 @@ call: (`import http.client` 7.2 s, now ~0.5 s); sources plus bytecode would exceed the crates.io 10 MiB crate cap. Modules that cannot work in the guest (FFI, TLS, sockets, TTY) and pure-Python twins of C modules are - not shipped (L-CPY-009). + not shipped (L-CPY-009). `pdb` is a stand-in (`guest/pdb.py`): doctest + imports it, `breakpoint()` prints a notice and continues. +9. **xz snapshot artifact**: `artifacts/python.wasm.xz` (xz -9e) instead of + gzip: 4.99 MB vs ~6.7 MB for the same snapshot. `build.rs` decodes it + with `lzma-rs` (pure Rust build-dependency; ~2.5 s in a debug build + script, nothing at run time). Rejected: zstd/brotli (C or no pure-Rust + encoder parity) and splitting the stdlib into a second crate (single + companion crate is a product decision). +10. **Preload budget**: what the snapshot imports is chosen by per-call + import cost on Pulley against crate size. Preloaded beyond the core set: + bashkit's requests/httpx, `urllib.request` (with http.client and email, + 0.47 s per call before), `tomllib`, `configparser`, + `xml.etree.ElementTree` (0.1-0.12 s each). Each costs snapshot bytes, + not per-call time (`print(1)` stayed ~4.1 ms). `mailbox` is not shipped. Measured on a 4-vCPU x86-64 VM (see `criterion-python-*` results under `crates/bashkit/benches/results/`): first `python3` in a fresh process @@ -80,7 +93,9 @@ Measured on a 4-vCPU x86-64 VM (see `criterion-python-*` results under CPU-bound Python is ~4-30x slower than Monty. 1024 concurrent tenants × 4 calls: 0 failures, ~165 calls/s on 4 vCPUs, 3.0 GB peak RSS (2026-10-07, after immortal objects; was ~90 calls/s, 4.7 GB). Importing a module outside -the snapshot costs ~0.27 s (`email.message`) to ~0.46 s (`http.client`). +the snapshot costs ~0.1-0.2 s (`unittest` 0.2 s, `zoneinfo` 0.11 s, +`tarfile` 0.1 s); `email`, `http.client` and `urllib.request` are preloaded +(~0 per call, was 0.27-0.47 s). ### Native code (opt-in, `cpython-native`) @@ -262,7 +277,7 @@ CI: the Test job runs `cargo test -p bashkit --features cpython,http_client `crates/bashkit-cpython-wasm/guest/build.sh [WORK_DIR]` downloads pinned sources (CPython, WASI SDK, wasmtime CLI for `wizer`, zlib, sqlite), builds, snapshots, smoke-tests with the reference runtime and rewrites `artifacts/` -(`python.wasm.gz`, `python314.zip`, `MANIFEST` with checksums). Commit all +(`python.wasm.xz`, `python314.zip`, `MANIFEST` with checksums). Commit all three together. Changing `engine_config()` or the wasmtime version requires no guest rebuild; `build.rs` recompiles the `.cwasm`. diff --git a/supply-chain/config.toml b/supply-chain/config.toml index a7b15539c..85b1cc318 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -388,6 +388,14 @@ criteria = "safe-to-deploy" version = "0.3.0" criteria = "safe-to-deploy" +[[exemptions.crc]] +version = "3.4.0" +criteria = "safe-to-deploy" + +[[exemptions.crc-catalog]] +version = "2.5.0" +criteria = "safe-to-deploy" + [[exemptions.crc32c]] version = "0.6.8" criteria = "safe-to-deploy" @@ -1017,6 +1025,10 @@ criteria = "safe-to-deploy" version = "0.7.2" criteria = "safe-to-deploy" +[[exemptions.lzma-rs]] +version = "0.3.0" +criteria = "safe-to-deploy" + [[exemptions.mach2]] version = "0.6.0" criteria = "safe-to-deploy"