|
| 1 | +// Manual browser experiment. CI gates deterministic bytes/semantics, not noisy wall time. |
| 2 | +// Supply PLAYWRIGHT_MODULE when Playwright is installed outside this workspace. |
| 3 | +import { createServer } from "node:http"; |
| 4 | +import { readFileSync } from "node:fs"; |
| 5 | +import { resolve } from "node:path"; |
| 6 | +import { pathToFileURL } from "node:url"; |
| 7 | +import { brotliCompressSync, constants } from "node:zlib"; |
| 8 | +import { artifact } from "./artifact.mjs"; |
| 9 | + |
| 10 | +const { chromium } = await import(process.env.PLAYWRIGHT_MODULE ?? "playwright"); |
| 11 | +const packages = process.argv.slice(2).map(resolvePath => resolve(resolvePath)); |
| 12 | +if (!packages.length) throw new Error("pass package directories to compare"); |
| 13 | +const trials = Number(process.env.TRIALS ?? 5); |
| 14 | +if (!Number.isInteger(trials) || trials < 1 || trials > 20) throw new Error("TRIALS must be 1..20"); |
| 15 | +const conditions = { |
| 16 | + local: null, |
| 17 | + fast4g: { downloadThroughput: 1012500, uploadThroughput: 168750, latency: 165 }, |
| 18 | + slow4g: { downloadThroughput: 180000, uploadThroughput: 84375, latency: 562.5 }, |
| 19 | +}; |
| 20 | +const routes = new Map(); |
| 21 | +packages.forEach((pkg, i) => { |
| 22 | + for (const file of ["index.js", "bashkit_wasm.js", "bashkit_wasm_bg.wasm"]) { |
| 23 | + routes.set(`/${i}/${file}`, { type: file.endsWith("wasm") ? "application/wasm" : "text/javascript", |
| 24 | + bytes: brotliCompressSync(readFileSync(resolve(pkg, file)), { params: { [constants.BROTLI_PARAM_QUALITY]: 11 } }) }); |
| 25 | + } |
| 26 | + const html = `<pre id="screen"></pre><script type="module"> |
| 27 | + const marks = { start: performance.now() }; |
| 28 | + const { initBashkit, Bash, Terminal } = await import('/${i}/index.js'); |
| 29 | + marks.imported = performance.now(); |
| 30 | + await initBashkit(); marks.initialized = performance.now(); |
| 31 | + const bash = new Bash(); |
| 32 | + const r = await bash.execute('echo hi'); |
| 33 | + if (r.stdout !== 'hi\\n' || r.exitCode !== 0) throw new Error('first command failed'); |
| 34 | + marks.command = performance.now(); |
| 35 | + if (Terminal) { |
| 36 | + const terminal = new Terminal(); await terminal.runUntilIdle(); |
| 37 | + terminal.send('echo hi\\r'); await terminal.runUntilIdle(); |
| 38 | + document.querySelector('#screen').textContent = terminal.screenText(); |
| 39 | + if (!terminal.screenText().includes('hi\\n$')) throw new Error('terminal failed'); |
| 40 | + } else document.querySelector('#screen').textContent = r.stdout + '$'; |
| 41 | + await new Promise(requestAnimationFrame); await new Promise(requestAnimationFrame); |
| 42 | + marks.prompt = performance.now(); |
| 43 | + window.result = { marks, resources: performance.getEntriesByType('resource').map(r => ({ name: r.name.split('/').at(-1), start: r.startTime, end: r.responseEnd, transfer: r.transferSize, encoded: r.encodedBodySize })) }; |
| 44 | + </script>`; |
| 45 | + routes.set(`/${i}/`, { type: "text/html", bytes: Buffer.from(html), plain: true }); |
| 46 | +}); |
| 47 | +const server = createServer((req, res) => { |
| 48 | + const route = routes.get(req.url); |
| 49 | + if (!route) { res.writeHead(404); res.end(); return; } |
| 50 | + res.writeHead(200, { "Content-Type": route.type, "Content-Length": route.bytes.length, |
| 51 | + "Cache-Control": "public, max-age=31536000, immutable", "Vary": "Accept-Encoding", |
| 52 | + ...(route.plain ? {} : { "Content-Encoding": "br" }) }); |
| 53 | + res.end(route.bytes); |
| 54 | +}); |
| 55 | +await new Promise(r => server.listen(0, "127.0.0.1", r)); |
| 56 | +let browser; |
| 57 | +try { |
| 58 | + browser = await chromium.launch({ channel: process.env.CHROME_CHANNEL ?? "chrome", headless: true }); |
| 59 | + const samples = []; |
| 60 | + for (const [condition, throttle] of Object.entries(conditions)) { |
| 61 | + for (let trial = 0; trial < trials; trial++) { |
| 62 | + const order = packages.map((_, i) => (i + trial) % packages.length); |
| 63 | + if (trial % 2) order.reverse(); |
| 64 | + for (const i of order) { |
| 65 | + const context = await browser.newContext(); // Cold HTTP cache per sample. |
| 66 | + try { |
| 67 | + const page = await context.newPage(); |
| 68 | + const session = await context.newCDPSession(page); |
| 69 | + await session.send("Network.enable"); |
| 70 | + if (throttle) await session.send("Network.emulateNetworkConditions", { offline: false, ...throttle }); |
| 71 | + await session.send("Emulation.setCPUThrottlingRate", { rate: 4 }); |
| 72 | + const errors = []; page.on("pageerror", e => errors.push(String(e))); |
| 73 | + await page.goto(`http://127.0.0.1:${server.address().port}/${i}/`, { waitUntil: "domcontentloaded", timeout: 60000 }); |
| 74 | + await page.waitForFunction(() => window.result, null, { timeout: 60000 }); |
| 75 | + const result = await page.evaluate(() => window.result); |
| 76 | + if (errors.length) throw new Error(errors.join("\n")); |
| 77 | + samples.push({ package: i, condition, trial, ...result }); |
| 78 | + console.error(`${condition} ${trial + 1}/${trials} package ${i}: ${result.marks.prompt.toFixed(1)}ms`); |
| 79 | + } finally { await context.close(); } |
| 80 | + } |
| 81 | + } |
| 82 | + } |
| 83 | + console.log(JSON.stringify({ method: "Cold context, Brotli 11, CDP Fast/Slow 4G, page CPU 4x; main-thread JS-host WASM, no terminal renderer dependency; prompt after first echo and two animation frames", browser: browser.version(), node: process.version, |
| 84 | + packages: packages.map(pkg => ({ path: pkg, ...artifact(pkg) })), samples }, null, 2)); |
| 85 | +} finally { await browser?.close(); await new Promise(r => server.close(r)); } |
0 commit comments