feat(cpython): xz snapshot, preload urllib.request and parsers, fix doctest

crates/bashkit-cpython-wasm/Cargo.toml

@@ -41,5 +41,6 @@ native = []
 
 [build-dependencies]
 wasmtime = { workspace = true, features = ["cranelift", "parallel-compilation"] }
-flate2 = { workspace = true }
+# Decodes artifacts/python.wasm.xz (pure Rust, build time only).
+lzma-rs = "0.3"
 

crates/bashkit-cpython-wasm/THIRD_PARTY_LICENSES.md

@@ -7,14 +7,14 @@ libraries statically linked into it (see `guest/build.sh` and
 
 | Component | Version | License | Where |
 |-----------|---------|---------|-------|
-| CPython (interpreter and stdlib) | 3.14.8 | Python-2.0 (PSF License Agreement and predecessors) | `artifacts/python.wasm.gz`, `artifacts/python314.zip` |
-| libmpdec (`decimal`) | bundled with CPython | BSD-2-Clause | `artifacts/python.wasm.gz` |
-| Expat (`pyexpat`, `xml.etree`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` |
-| HACL* (`hashlib` md5/sha1/sha2/sha3/blake2, `hmac`) | bundled with CPython | MIT | `artifacts/python.wasm.gz` |
-| zlib | 1.3.1 | Zlib | `artifacts/python.wasm.gz` |
-| SQLite | 3.50.4 | Public domain (blessing) | `artifacts/python.wasm.gz` |
-| wasi-libc (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT; musl parts MIT | `artifacts/python.wasm.gz` |
-| compiler-rt builtins (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception | `artifacts/python.wasm.gz` |
+| CPython (interpreter and stdlib) | 3.14.8 | Python-2.0 (PSF License Agreement and predecessors) | `artifacts/python.wasm.xz`, `artifacts/python314.zip` |
+| libmpdec (`decimal`) | bundled with CPython | BSD-2-Clause | `artifacts/python.wasm.xz` |
+| Expat (`pyexpat`, `xml.etree`) | bundled with CPython | MIT | `artifacts/python.wasm.xz` |
+| HACL* (`hashlib` md5/sha1/sha2/sha3/blake2, `hmac`) | bundled with CPython | MIT | `artifacts/python.wasm.xz` |
+| zlib | 1.3.1 | Zlib | `artifacts/python.wasm.xz` |
+| SQLite | 3.50.4 | Public domain (blessing) | `artifacts/python.wasm.xz` |
+| wasi-libc (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT; musl parts MIT | `artifacts/python.wasm.xz` |
+| compiler-rt builtins (WASI SDK 24) | 24 | Apache-2.0 WITH LLVM-exception | `artifacts/python.wasm.xz` |
 
 The full license texts that require reproduction follow.
 

crates/bashkit-cpython-wasm/artifacts/MANIFEST

@@ -3,5 +3,5 @@ wasi_sdk=24
 wasmtime_wizer=49.0.1
 zlib=1.3.1
 sqlite=sqlite-amalgamation-3500400
-31c43d757812151582ff3e274f0d9c65cf210064a46646a5a73a68704b83329b  python.wasm.gz
-7c1fec908a3c250e0591a4892f3dfbaa73e5c81798dfa33b1dc80e18931dbb74  python314.zip
+77058ae157db6f10e3fe9e73745147627a05bca03ee6a6c50a0a5f04ea1c7fa6  python.wasm.xz
+bfe72099bf1ab706b9de9a8856f509ac5db4fb1c86c29a49c622d37490ce388e  python314.zip

crates/bashkit-cpython-wasm/build.rs

@@ -5,15 +5,14 @@
 //! product feature). Set `BASHKIT_CPYTHON_CWASM=/path/python.cwasm` to reuse a
 //! module precompiled by the same wasmtime version and configuration.
 
-use std::io::Read;
 use std::path::PathBuf;
 
 include!("src/config.rs");
 
 fn main() {
     let manifest = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap());
     let out = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("python.cwasm");
-    let snapshot = manifest.join("artifacts/python.wasm.gz");
+    let snapshot = manifest.join("artifacts/python.wasm.xz");
     println!("cargo:rerun-if-changed={}", snapshot.display());
     println!("cargo:rerun-if-changed=src/config.rs");
     println!("cargo:rerun-if-env-changed=BASHKIT_CPYTHON_CWASM");
@@ -23,9 +22,11 @@ fn main() {
         return;
     }
 
+    // xz, not gzip: ~30% smaller, which keeps the crate under crates.io's
+    // 10 MiB cap with more modules preloaded. Decoded here only.
     let mut wasm = Vec::new();
-    flate2::read::GzDecoder::new(std::fs::File::open(&snapshot).expect("open snapshot"))
-        .read_to_end(&mut wasm)
+    let file = std::fs::File::open(&snapshot).expect("open snapshot");
+    lzma_rs::xz_decompress(&mut std::io::BufReader::new(file), &mut wasm)
         .expect("decompress snapshot");
     // The build script's own cfg is the host's; read the crate feature and
     // the cross-compilation target from Cargo instead.

crates/bashkit-cpython-wasm/guest/_bashkit_boot.py

@@ -47,6 +47,8 @@
     # bashkit's own HTTP clients over the host bridge: preloaded so
     # `import requests` / `import httpx` cost nothing per call.
     "requests", "requests.utils", "httpx", "httpx2",
+    # Common in agent scripts and 0.1-0.5 s each to import on Pulley.
+    "urllib.request", "tomllib", "configparser", "xml.etree.ElementTree",
 )
 
 _USAGE = """\

crates/bashkit-cpython-wasm/guest/build.sh

@@ -11,11 +11,12 @@
 # - The stdlib ships as one deflate-compressed zip of .py sources. The
 #   snapshot caches the zip directory, so the runtime must serve the exact
 #   same zip bytes; both artifacts are produced together by this script.
-# - Output is a gzip of the stripped snapshot plus the zip. The crate's
+# - Output is an xz of the stripped snapshot plus the zip (xz, not gzip:
+#   ~30% smaller, which keeps the crate under the crates.io 10 MiB cap). The crate's
 #   build.rs compiles the snapshot to Pulley bytecode ahead of time.
 #
 # Usage: guest/build.sh [WORK_DIR]
-# Requires: curl, tar, unzip, python3 (>=3.11), make, a C compiler for the
+# Requires: curl, tar, unzip, xz, python3 (>=3.11), make, a C compiler for the
 # native build Python. Everything else is downloaded into WORK_DIR.
 set -euo pipefail
 
@@ -130,11 +131,13 @@ cp -r "$HERE/requests" "$WORK/stdlib/requests"
     cd "$WORK/stdlib"
     # Not usable or not useful in a sandboxed, single-threaded, headless guest.
     rm -rf test idlelib tkinter turtledemo ensurepip venv pydoc_data turtle.py \
-        _pyrepl/__pycache__ curses dbm/gnu.py dbm/ndbm.py multiprocessing concurrent/futures/process.py
+        _pyrepl/__pycache__ curses mailbox.py dbm/gnu.py dbm/ndbm.py multiprocessing concurrent/futures/process.py
     # No FFI, TLS, sockets or TTY in the guest: these can never work.
     rm -rf ctypes ssl.py ftplib.py imaplib.py poplib.py smtplib.py socketserver.py \
         http/server.py wsgiref xmlrpc webbrowser.py _pyrepl pdb.py bdb.py pydoc.py \
         _aix_support.py _android_support.py _ios_support.py _osx_support.py
+    # Debugger stand-in: doctest imports pdb; breakpoint() prints a notice.
+    cp "$HERE/pdb.py" pdb.py
     # HTTP goes through the host's egress pipeline, not sockets
     # (_bashkit_http.py); http.client patches itself when first imported.
     printf '\n# bashkit: connections go through the host (see _bashkit_http).\nimport _bashkit_http\n_bashkit_http.patch_http_client(globals())\ndel _bashkit_http\n' >>http/client.py
@@ -185,7 +188,8 @@ case "$out" in
 esac
 
 mkdir -p "$CRATE/artifacts"
-gzip -9 -n -c "$WORK/python.wasm" >"$CRATE/artifacts/python.wasm.gz"
+rm -f "$CRATE/artifacts/python.wasm.gz"
+xz -9e -T1 -c "$WORK/python.wasm" >"$CRATE/artifacts/python.wasm.xz"
 cp "$ROOT/usr/local/lib/python314.zip" "$CRATE/artifacts/python314.zip"
 (
     cd "$CRATE/artifacts"
@@ -195,7 +199,7 @@ cp "$ROOT/usr/local/lib/python314.zip" "$CRATE/artifacts/python314.zip"
         echo "wasmtime_wizer=${WASMTIME_VERSION}"
         echo "zlib=${ZLIB_VERSION}"
         echo "sqlite=${SQLITE_AMALGAMATION}"
-        sha256sum python.wasm.gz python314.zip
+        sha256sum python.wasm.xz python314.zip
     } >MANIFEST
 )
 ls -la "$CRATE/artifacts"

crates/bashkit-cpython-wasm/guest/pdb.py

@@ -0,0 +1,46 @@
+"""pdb stand-in for bashkit's CPython guest.
+
+There is no interactive terminal in the sandbox, so the real debugger is
+not shipped. This stub keeps modules that import pdb (doctest, unittest
+helpers) working: `breakpoint()` / `pdb.set_trace()` print a notice and
+continue; `pdb.run*` raise RuntimeError.
+"""
+
+import sys
+
+_MSG = "pdb: debugger not available in the bashkit sandbox; continuing"
+
+
+class Pdb:
+    """Constructible (doctest creates one per run); debugging is a no-op."""
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def set_trace(self, *args, **kwargs):
+        print(_MSG, file=sys.stderr)
+
+    def reset(self):
+        pass
+
+    def set_continue(self):
+        pass
+
+
+def set_trace(*args, header=None, **kwargs):
+    print(_MSG, file=sys.stderr)
+
+
+def post_mortem(*args, **kwargs):
+    print(_MSG, file=sys.stderr)
+
+
+def pm():
+    post_mortem()
+
+
+def run(*args, **kwargs):
+    raise RuntimeError("pdb is not available in the bashkit sandbox")
+
+
+runeval = runcall = runctx = run

crates/bashkit/docs/cpython.md

@@ -203,7 +203,8 @@ with httpx.Client(base_url="https://api.example.com", headers={"X-Key": "..."})
   `_hashlib` (OpenSSL), `tkinter`, `curses`, `readline`, `dbm.gnu` are not
   built. `hashlib` still provides md5, sha1, sha2, sha3 and blake2.
 - **No interactive mode**: `python3` with no program reads one from stdin;
-  there is no REPL, and `pdb` and `pydoc` (`help()`) are not shipped.
+  there is no REPL, `pydoc` (`help()`) is not shipped, and `breakpoint()`
+  prints a notice and continues (no debugger).
 - **Stdlib is bytecode only**: tracebacks through stdlib code show no source
   line, and `inspect.getsource()` fails on stdlib objects. Your own code
   keeps full tracebacks. Non-HTTP network clients and servers (`smtplib`,

crates/bashkit/tests/integration/cpython_http_tests.rs

@@ -743,7 +743,7 @@ except requests.ConnectionError as e:
 import sys
 before = set(sys.modules)
 import requests, httpx, httpx2
-heavy = {"http.client", "email", "urllib3", "ssl", "http.cookiejar"}
+heavy = {"urllib3", "ssl", "http.cookiejar", "charset_normalizer"}
 print(sorted(set(sys.modules) - before), sorted(heavy & set(sys.modules)))
 "#,
         )