From c5ae2976691ef2283ea73fd5add78dd9ba656fe7 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 29 Oct 2026 14:51:58 +0000 Subject: [PATCH 1/2] feat(examples): add script-only JSON pipeline example --- .github/workflows/main.yml | 18 +- examples/components/README.md | 1 + .../script-only-pipeline/.gitignore | 2 + .../components/script-only-pipeline/README.md | 236 ++++++++++++++++++ .../components/dataset/dataset.ts | 43 ++++ .../components/geoip/geoip.ts | 35 +++ .../components/risk-score/risk-score.ts | 35 +++ .../components/shared/state.ts | 34 +++ .../components/top-pages/top-pages.ts | 29 +++ .../demo/access-logs.json | 14 ++ .../script-only-pipeline/demo/browser.js | 228 +++++++++++++++++ .../demo/custom-component.js | 70 ++++++ .../demo/custom-template.ts | 21 ++ .../script-only-pipeline/demo/editor.js | 28 +++ .../script-only-pipeline/demo/index.html | 91 +++++++ .../script-only-pipeline/demo/node.mjs | 41 +++ .../demo/shared/pipeline.js | 47 ++++ .../demo/shared/read-script.js | 27 ++ .../script-only-pipeline/demo/style.css | 233 +++++++++++++++++ .../script-only-pipeline/demo/vite.config.js | 19 ++ .../example-log-pipeline-dataset.d.ts | 19 ++ .../example-log-pipeline-processor.d.ts | 12 + .../generated/types/pipeline.d.ts | 6 + .../script-only-pipeline/package.json | 30 +++ .../scripts/transpile.mjs | 20 ++ .../script-only-pipeline/test/browser.mjs | 214 ++++++++++++++++ .../script-only-pipeline/test/pipeline.mjs | 154 ++++++++++++ .../script-only-pipeline/tsconfig.json | 15 ++ .../script-only-pipeline/wit/pipeline.wit | 35 +++ pnpm-lock.yaml | 19 ++ pnpm-workspace.yaml | 1 + 31 files changed, 1769 insertions(+), 8 deletions(-) create mode 100644 examples/components/script-only-pipeline/.gitignore create mode 100644 examples/components/script-only-pipeline/README.md create mode 100644 examples/components/script-only-pipeline/components/dataset/dataset.ts create mode 100644 examples/components/script-only-pipeline/components/geoip/geoip.ts create mode 100644 examples/components/script-only-pipeline/components/risk-score/risk-score.ts create mode 100644 examples/components/script-only-pipeline/components/shared/state.ts create mode 100644 examples/components/script-only-pipeline/components/top-pages/top-pages.ts create mode 100644 examples/components/script-only-pipeline/demo/access-logs.json create mode 100644 examples/components/script-only-pipeline/demo/browser.js create mode 100644 examples/components/script-only-pipeline/demo/custom-component.js create mode 100644 examples/components/script-only-pipeline/demo/custom-template.ts create mode 100644 examples/components/script-only-pipeline/demo/editor.js create mode 100644 examples/components/script-only-pipeline/demo/index.html create mode 100644 examples/components/script-only-pipeline/demo/node.mjs create mode 100644 examples/components/script-only-pipeline/demo/shared/pipeline.js create mode 100644 examples/components/script-only-pipeline/demo/shared/read-script.js create mode 100644 examples/components/script-only-pipeline/demo/style.css create mode 100644 examples/components/script-only-pipeline/demo/vite.config.js create mode 100644 examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts create mode 100644 examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts create mode 100644 examples/components/script-only-pipeline/generated/types/pipeline.d.ts create mode 100644 examples/components/script-only-pipeline/package.json create mode 100644 examples/components/script-only-pipeline/scripts/transpile.mjs create mode 100644 examples/components/script-only-pipeline/test/browser.mjs create mode 100644 examples/components/script-only-pipeline/test/pipeline.mjs create mode 100644 examples/components/script-only-pipeline/tsconfig.json create mode 100644 examples/components/script-only-pipeline/wit/pipeline.wit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9d8a3a737..a75493d7f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -804,15 +804,22 @@ jobs: setup-std: true - name: node-sveltekit-server workspace: examples/components/node-sveltekit-server + uses-browser: true setup-p2-shim: true setup-std: true - name: fs-write-file workspace: examples/components/fs-write-file - name: browser-p2-shims workspace: examples/components/browser-p2-shims + uses-browser: true setup-p2-shim: true + - name: script-only-pipeline + workspace: examples/components/script-only-pipeline + uses-browser: true - name: native-messaging workspace: examples/components/native-messaging + uses-browser: true + uses-firefox: true setup-p2-shim: true - name: wintertc workspace: examples/components/wintertc @@ -900,23 +907,18 @@ jobs: - run: pnpm install - name: Cache example browsers - if: ${{ matrix.project.name == 'browser-p2-shims' || matrix.project.name == 'native-messaging' }} + if: ${{ matrix.project.uses-browser }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/puppeteer key: puppeteer-examples-${{ runner.os }}-${{ runner.arch }}-${{ matrix.project.name }}-${{ hashFiles('pnpm-lock.yaml', 'scripts/install-puppeteer.mjs') }} - name: Install Puppeteer browser - if: ${{ matrix.project.name == 'browser-p2-shims' || matrix.project.name == 'native-messaging' }} + if: ${{ matrix.project.uses-browser }} run: pnpm run test:setup:puppeteer - - name: Install SvelteKit example browser - if: ${{ matrix.project.name == 'node-sveltekit-server' }} - working-directory: ${{ matrix.project.workspace }} - run: pnpm run setup:browser - - name: Install Firefox - if: ${{ matrix.project.name == 'native-messaging' }} + if: ${{ matrix.project.uses-firefox }} run: pnpm run test:setup:firefox - name: Build required deps diff --git a/examples/components/README.md b/examples/components/README.md index a2ccdfb90..4da5214f3 100644 --- a/examples/components/README.md +++ b/examples/components/README.md @@ -21,6 +21,7 @@ A brief description of the examples contained in this folder: | [`native-messaging`](./native-messaging) | Runs a Jco-built WASI command as a Firefox and Chromium native-messaging host | | [`node-mcp-server`](./node-mcp-server) | MCP 2026-07-28 stateless server: the same source runs in Node.js and as a component using jco-std | | [`p3-stream-echo-qjs`](./p3-stream-echo-qjs) | Echoes a Preview 3 stream through a JavaScript component built with the QuickJS backend | +| [`script-only-pipeline`](./script-only-pipeline) | TypeScript script-only components share one engine and JSON dataset in Node.js and an uploadable browser pipeline | | [`node-sveltekit-server`](./node-sveltekit-server) | SvelteKit TODO app: the Node adapter output runs in Node.js and as a component using jco-std | | [`string-reverse-upper`](./string-reverse-upper) | `import`s functionality to build more advanced computation to `export` | | [`string-reverse`](./string-reverse) | `export`s basic functionality with a slightly more involved WIT interface and more complex types | diff --git a/examples/components/script-only-pipeline/.gitignore b/examples/components/script-only-pipeline/.gitignore new file mode 100644 index 000000000..3d2bc6269 --- /dev/null +++ b/examples/components/script-only-pipeline/.gitignore @@ -0,0 +1,2 @@ +/dist/ +/node_modules/ diff --git a/examples/components/script-only-pipeline/README.md b/examples/components/script-only-pipeline/README.md new file mode 100644 index 000000000..2aa56e296 --- /dev/null +++ b/examples/components/script-only-pipeline/README.md @@ -0,0 +1,236 @@ +# Script-only JSON pipeline example + +This example builds TypeScript [script-only components](../../../docs/src/advanced/script-only-components.md) +with Jco and runs them in **Node.js and the browser using the same pipeline code**. A single QuickJS +engine holds a shared dataset of synthetic access logs. Small `.wasm` components supply the scripts +that filter those logs, calculate risk scores, and count page visits. + +The browser starts with an empty pipeline. Download the example components from the page, upload them +to add steps, edit their JSON options, and reorder or remove them to see how the results change. +Uploads are read and transpiled locally in the browser; there is no processing server. +You can also write your own TypeScript processor in the page's editor, build a script-only component, +add it to the pipeline, and download the resulting `.wasm` for either host. + +## Quickstart + +From this directory in a [built Jco checkout](../../../docs/src/contributing.md#building): + +```console +pnpm install +pnpm run all +pnpm run demo:open +``` + +Use Node.js 24 or newer. This example uses the workspace version of Jco, which includes +`componentize --script-only`. If Puppeteer has no browser installed, run +`pnpm run test:setup:puppeteer` from the repository root before `pnpm run all`. + +`pnpm run all` follows the other examples: generate types, check TypeScript, build the components, +transpile the engine and dataset initializer, run the Node.js demo, and run Node.js and Puppeteer tests. +The QuickJS engine uses a synchronous ABI and stubbed WASI because these processors only operate on +in-memory JSON. Neither host needs JSPI flags, filesystem access from the guest, or WASI shims. + +## Try the browser demo + +Open `http://localhost:5173/demo/` after `pnpm run demo`, or use `pnpm run demo:open`. + +1. Run the empty pipeline: all 12 input logs appear unchanged. +2. Download **GeoIP filter**, **Risk score**, and **Most visited pages** using the page's links. +3. Upload the three `.wasm` files in that order. You can also select them directly from `dist/components/`. +4. Run the pipeline. The default GeoIP filter keeps five German logs, the risk step annotates them, + and the page counter reports `/docs: 2`, `/login: 2`, and `/admin: 1`. +5. Set the risk step's options to `{"minScore":70}` and rerun. Three logs remain, with scores 70, 70, and 90. +6. Move the page counter to the beginning. Its report now counts all 12 input logs, before filtering. + +The JSON input is editable. Every run starts from that input, so a previous run's filters, scores, and +reports never leak into the next run. **Restore sample** resets the input; **Clear pipeline** removes +the steps. Invalid uploads or options show an error on the page and can be corrected without a reload. + +The IP addresses come from documentation ranges. The US/DE/JP mappings are invented, and the risk +score is an illustrative rule: 20 points for a 4xx response or 40 for a 5xx, 30 for `/admin` or `/login`, +and 20 when the current dataset contains at least three requests from the same IP, capped at 100. + +### Create a component in the browser + +The **Write your own component** section starts with an editable HTTP status filter. Its +[CodeMirror](https://codemirror.net/examples/basic/) editor highlights TypeScript. Change the code, +give it a name, and choose **Create component & add step**. Run the pipeline to try it, or use +**Download .wasm** to save the generated component. That file can be uploaded again, or copied to +`dist/components/` and selected by name in the Node.js demo. + +Custom processors export the same `processor` interface as the checked-in components. Import +`getDataset()` and `parseOptions()` from `pipeline:state`; the browser bundles the existing +`components/shared/state.ts` implementation into your script. `getDataset()` provides mutable `logs` +and `reports`, and `describe()` supplies the step name, description, and default JSON options. +Runtime imports other than `pipeline:state` are not available in this demo. + +The browser uses [esbuild's WebAssembly build](https://esbuild.github.io/api/#browser) to bundle +TypeScript into an ES module with a trailing export list, then calls Jco's +`scripts.createComponent()` API from `@bytecodealliance/jco-transpile/component` to package the source. +It reads that Wasm back through the same loader as an upload and validates `describe()` before adding +the step. Code runs inside the shared QuickJS engine. The JavaScript engine is reused as you build +and load new components. + +The editor and compiler load from versioned CDN URLs, so browser authoring needs internet access. +Your source stays in the browser; no compilation requests are sent to a server. If the editor CDN is +unavailable, the page keeps a plain source text area. Build errors appear beside the editor and leave +the pipeline intact. Browser builds validate syntax and exports; full TypeScript checking for the +checked-in components and editor template is provided by `pnpm run check`. + +## Project layout + +```text +components/ + dataset/dataset.ts Validate/reset input and read snapshots + geoip/geoip.ts Annotate synthetic countries and filter + risk-score/risk-score.ts Score and optionally filter requests + top-pages/top-pages.ts Add a page-count report + shared/state.ts Shared in-engine data and option parsing +wit/pipeline.wit Dataset and processor interfaces +generated/types/ Guest declarations generated from WIT +scripts/transpile.mjs Transpile the engine and dataset initializer +demo/ + shared/pipeline.js Engine lifecycle and pipeline execution for both hosts + shared/read-script.js Runtime .wasm transpilation and script metadata extraction + node.mjs Node.js entry point + browser.js Uploads, pipeline editing, and results + custom-component.js In-browser TypeScript bundling and script-only Wasm creation + custom-template.ts Editable, type-checked starting processor + editor.js CDN-loaded CodeMirror editor + access-logs.json The same sample input for both hosts + index.html, style.css Browser interface + vite.config.js Local demo server +test/ Node.js and real browser integration tests +dist/ Generated component binaries and transpiled bindings (ignored) +``` + +## Step-by-step + +### Generate interface types + +```console +pnpm run gen:types +pnpm run check +``` + +`jco guest-types` generates declarations from [`wit/pipeline.wit`](./wit/pipeline.wit). Each component's +export is checked against its generated interface, for example: + +```ts +import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; + +export const processor: typeof ProcessorInterface = { + describe() { + /* name, description, defaultOptions */ + }, + process(options) { + /* update shared data and return a summary */ + }, +}; +``` + +The generated `AccessLog` type also describes the shared records. These declarations are checked in +for editor support and regenerated by `build`; Jco bundles TypeScript directly, so there is no separate +JavaScript compilation step. + +### Build the components + +```console +pnpm run build:component +``` + +The command passes all four TypeScript entry points and the WIT world to +`jco componentize --script-only`. It writes: + +| Artifact | Purpose | +| --------------------------------- | -------------------------------------------------------------------- | +| `dist/components/engine.wasm` | One QuickJS runtime implementing the WIT world and script loader | +| `dist/components/engine.wit` | The merged engine world, including `jco:script-engine/script-loader` | +| `dist/components/dataset.wasm` | Script that initializes and snapshots shared data | +| `dist/components/geoip.wasm` | Uploadable location filter | +| `dist/components/risk-score.wasm` | Uploadable request scorer | +| `dist/components/top-pages.wasm` | Uploadable page counter | + +Each script-only component exports `jco:script/script`, which returns bundled source and metadata. +It carries **no JavaScript engine**. The dataset script and processors implement different subsets of +the engine's exports; the host calls only the interface supplied by the currently loaded script. + +### Transpile and run + +```console +pnpm run transpile +pnpm run run +``` + +The engine and dataset initializer are transpiled ahead of time with `--instantiation async`. +Processing components remain `.wasm` files until the host loads them. The shared +[`readScript`](./demo/shared/read-script.js) helper uses `@bytecodealliance/jco/component` to transpile +those files, instantiates their generated bindings, and reads `script.name()`, `script.source()`, +`script.exports()`, and `script.engineWorld()`. + +The shared [`createPipeline`](./demo/shared/pipeline.js) runner instantiates the engine once. Its only +platform adapter is a function that reads bytes from a URL: Node.js supplies `readFile`, and the browser +supplies `fetch`. Both entry points use the same script reader and execution code. + +For each run, the runner loads the dataset script and calls `dataset.reset(input)`. For each step, it +loads that step's source with `scriptLoader.load`, calls `processor.process(options)`, then reloads the +dataset script to read a snapshot. Loading scripts replaces the active exports, but retains the +engine's JavaScript realm. The working dataset lives on that realm's `globalThis`, shared through +[`components/shared/state.ts`](./components/shared/state.ts); it is not passed back and forth between +processors as JSON. Only input, options, summaries, and output snapshots cross the host boundary. + +> [!NOTE] +> Scripts in one engine share a realm and can access each other's data. Use trusted scripts together; +> use a separate engine instance for a separate trust boundary. An upload's metadata checks are for +> compatibility, not a security sandbox. The engine world is fixed at build time: new scripts must +> implement this example's processor interface and shared dataset convention. + +The Node.js demo prints: + +```text +GeoIP: 12 → 5 logs (DE) +Risk: scored logs and kept 5 with score ≥ 0 +Pages: counted 5 logs across 3 paths +``` + +The final JSON follows those summaries. The browser shows the same summaries and JSON. + +Choose the Node.js stages and their execution order with positional arguments or the comma-separated +`PIPELINE_STAGES` environment variable: + +```console +pnpm run run top-pages geoip +PIPELINE_STAGES=geoip,top-pages pnpm run run +PIPELINE_STAGES= pnpm run run +``` + +Arguments take precedence over the environment variable. With neither specified, the demo runs +`geoip`, `risk-score`, then `top-pages`. An empty environment value runs an empty pipeline. Stage names +are the filenames in `dist/components/` without `.wasm`; unknown names report the available stages and +exit with a nonzero status. Each selected stage uses its own default options. + +The entry point exports an async `main()` and calls it only when executed directly, so importing +`demo/node.mjs` does not run the demo. + +### Add a processor + +Create another folder under `components/`, export a `processor` checked against the generated interface, +and use `getDataset()` from `components/shared/state.ts` to read or modify the shared logs and reports. +Give its entry point a unique filename and add it to `build:component`. Rebuild, then upload the new +`.wasm` file: the browser discovers its name, description, and default options through `describe()`. +The pipeline runner needs no changes for another processor following this contract. + +### Test + +```console +pnpm run test:node +pnpm run test:browser +``` + +The tests exercise the built Wasm components, shared data across script loads, ordering and options, +repeat runs, empty input, invalid uploads, and recovery from errors. Node.js tests also cover CLI and +environment stage selection, precedence, unknown names, and the direct-execution guard. The Puppeteer +test uploads actual `.wasm` files, edits the highlighted TypeScript editor, builds custom components, +loads the downloadable bytes in Node.js, uploads them again, and checks recovery from syntax errors. +The browser test needs access to the editor and compiler CDNs. Like the other browser examples, it supports `PUPPETEER_PATH` and +`TEST_PUPPETEER_LAUNCH_ARGS` for local browser configuration. diff --git a/examples/components/script-only-pipeline/components/dataset/dataset.ts b/examples/components/script-only-pipeline/components/dataset/dataset.ts new file mode 100644 index 000000000..0b3bf9b48 --- /dev/null +++ b/examples/components/script-only-pipeline/components/dataset/dataset.ts @@ -0,0 +1,43 @@ +import type { dataset as DatasetInterface } from 'example:log-pipeline/pipeline'; +import type { AccessLog } from 'example:log-pipeline/dataset'; +import { getDataset, resetDataset } from '../shared/state.ts'; + +export const dataset: typeof DatasetInterface = { + reset(json) { + let rows: unknown; + try { + rows = JSON.parse(json); + } catch { + throw 'Input must be valid JSON'; + } + if (!Array.isArray(rows)) { + throw 'Input must be a JSON array of access logs'; + } + const logs: AccessLog[] = rows.map((row, index) => { + if ( + !row || + typeof row !== 'object' || + !['timestamp', 'ip', 'method', 'path'].every((key) => typeof row[key] === 'string') || + !Number.isInteger(row.status) || + row.status < 100 || + row.status > 599 + ) { + throw `Log ${index + 1} needs timestamp, ip, method, path, and an HTTP status (100–599)`; + } + // Start each run from raw logs; annotations belong to the processors. + return { + timestamp: row.timestamp, + ip: row.ip, + method: row.method, + path: row.path, + status: row.status, + country: undefined, + riskScore: undefined, + }; + }); + resetDataset(logs); + }, + snapshot() { + return JSON.stringify(getDataset()); + }, +}; diff --git a/examples/components/script-only-pipeline/components/geoip/geoip.ts b/examples/components/script-only-pipeline/components/geoip/geoip.ts new file mode 100644 index 000000000..f18830331 --- /dev/null +++ b/examples/components/script-only-pipeline/components/geoip/geoip.ts @@ -0,0 +1,35 @@ +import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; +import { getDataset, parseOptions } from '../shared/state.ts'; + +// These documentation-only IP ranges have invented locations for this demo. +// No network lookup or real GeoIP database is involved. +const countries = new Map([ + ['192.0.2', 'US'], + ['198.51.100', 'DE'], + ['203.0.113', 'JP'], +]); + +export const processor: typeof ProcessorInterface = { + describe() { + return { + name: 'GeoIP filter', + description: 'Annotate synthetic IP locations and keep one country, or all countries.', + defaultOptions: JSON.stringify({ country: 'DE' }), + }; + }, + process(json) { + const { country = 'DE' } = parseOptions(json); + if (typeof country !== 'string' || !['all', 'US', 'DE', 'JP', 'unknown'].includes(country)) { + throw 'country must be all, US, DE, JP, or unknown'; + } + const data = getDataset(); + const before = data.logs.length; + data.logs = data.logs + .map((log) => ({ + ...log, + country: countries.get(log.ip.split('.').slice(0, 3).join('.')) ?? 'unknown', + })) + .filter((log) => country === 'all' || log.country === country); + return `GeoIP: ${before} → ${data.logs.length} logs (${country})`; + }, +}; diff --git a/examples/components/script-only-pipeline/components/risk-score/risk-score.ts b/examples/components/script-only-pipeline/components/risk-score/risk-score.ts new file mode 100644 index 000000000..c84abfa97 --- /dev/null +++ b/examples/components/script-only-pipeline/components/risk-score/risk-score.ts @@ -0,0 +1,35 @@ +import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; +import { getDataset, parseOptions } from '../shared/state.ts'; + +export const processor: typeof ProcessorInterface = { + describe() { + return { + name: 'Risk score', + description: 'Score errors, sensitive paths, and repeated IPs; optionally keep scores above a threshold.', + defaultOptions: JSON.stringify({ minScore: 0 }), + }; + }, + process(json) { + const { minScore = 0 } = parseOptions(json); + if (typeof minScore !== 'number' || !Number.isInteger(minScore) || minScore < 0 || minScore > 100) { + throw 'minScore must be an integer from 0 to 100'; + } + const data = getDataset(); + const requests = new Map(); + for (const log of data.logs) { + requests.set(log.ip, (requests.get(log.ip) ?? 0) + 1); + } + data.logs = data.logs + .map((log) => ({ + ...log, + riskScore: Math.min( + 100, + (log.status >= 500 ? 40 : log.status >= 400 ? 20 : 0) + + (/^\/(admin|login)(\/|$)/.test(log.path) ? 30 : 0) + + ((requests.get(log.ip) ?? 0) >= 3 ? 20 : 0), + ), + })) + .filter((log) => log.riskScore >= minScore); + return `Risk: scored logs and kept ${data.logs.length} with score ≥ ${minScore}`; + }, +}; diff --git a/examples/components/script-only-pipeline/components/shared/state.ts b/examples/components/script-only-pipeline/components/shared/state.ts new file mode 100644 index 000000000..a3ad4fbb1 --- /dev/null +++ b/examples/components/script-only-pipeline/components/shared/state.ts @@ -0,0 +1,34 @@ +import type { AccessLog } from 'example:log-pipeline/dataset'; + +export interface Dataset { + logs: AccessLog[]; + reports: Record; +} + +// Each script is a separate bundle, but all scripts run in the same engine realm. +// Keep the data on that realm, not in a module-local variable or in the host. +const realm = globalThis as typeof globalThis & { logPipelineDataset?: Dataset }; + +export function resetDataset(logs: AccessLog[]): void { + realm.logPipelineDataset = { logs, reports: {} }; +} + +export function getDataset(): Dataset { + if (!realm.logPipelineDataset) { + throw 'Initialize the dataset before running a processor'; + } + return realm.logPipelineDataset; +} + +export function parseOptions(json: string): Record { + let options: unknown; + try { + options = JSON.parse(json); + } catch { + throw 'Options must be valid JSON'; + } + if (!options || typeof options !== 'object' || Array.isArray(options)) { + throw 'Options must be a JSON object'; + } + return options as Record; +} diff --git a/examples/components/script-only-pipeline/components/top-pages/top-pages.ts b/examples/components/script-only-pipeline/components/top-pages/top-pages.ts new file mode 100644 index 000000000..eed8e70d7 --- /dev/null +++ b/examples/components/script-only-pipeline/components/top-pages/top-pages.ts @@ -0,0 +1,29 @@ +import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; +import { getDataset, parseOptions } from '../shared/state.ts'; + +export const processor: typeof ProcessorInterface = { + describe() { + return { + name: 'Most visited pages', + description: 'Count paths in the current logs and add a topPages report without removing logs.', + defaultOptions: JSON.stringify({ limit: 5 }), + }; + }, + process(json) { + const { limit = 5 } = parseOptions(json); + if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > 100) { + throw 'limit must be an integer from 1 to 100'; + } + const data = getDataset(); + const counts = new Map(); + for (const log of data.logs) { + counts.set(log.path, (counts.get(log.path) ?? 0) + 1); + } + const pages = [...counts] + .map(([path, visits]) => ({ path, visits })) + .sort((a, b) => b.visits - a.visits || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + .slice(0, limit); + data.reports.topPages = pages; + return `Pages: counted ${data.logs.length} logs across ${counts.size} paths`; + }, +}; diff --git a/examples/components/script-only-pipeline/demo/access-logs.json b/examples/components/script-only-pipeline/demo/access-logs.json new file mode 100644 index 000000000..b320a2d84 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/access-logs.json @@ -0,0 +1,14 @@ +[ + { "timestamp": "2026-01-15T10:00:00Z", "ip": "192.0.2.10", "method": "GET", "path": "/", "status": 200 }, + { "timestamp": "2026-01-15T10:00:01Z", "ip": "198.51.100.20", "method": "GET", "path": "/docs", "status": 200 }, + { "timestamp": "2026-01-15T10:00:02Z", "ip": "203.0.113.30", "method": "GET", "path": "/pricing", "status": 200 }, + { "timestamp": "2026-01-15T10:00:03Z", "ip": "198.51.100.99", "method": "POST", "path": "/login", "status": 401 }, + { "timestamp": "2026-01-15T10:00:04Z", "ip": "198.51.100.99", "method": "GET", "path": "/admin", "status": 403 }, + { "timestamp": "2026-01-15T10:00:05Z", "ip": "192.0.2.10", "method": "GET", "path": "/docs", "status": 200 }, + { "timestamp": "2026-01-15T10:00:06Z", "ip": "198.51.100.21", "method": "GET", "path": "/docs", "status": 200 }, + { "timestamp": "2026-01-15T10:00:07Z", "ip": "203.0.113.31", "method": "GET", "path": "/", "status": 200 }, + { "timestamp": "2026-01-15T10:00:08Z", "ip": "198.51.100.99", "method": "POST", "path": "/login", "status": 500 }, + { "timestamp": "2026-01-15T10:00:09Z", "ip": "192.0.2.11", "method": "GET", "path": "/pricing", "status": 200 }, + { "timestamp": "2026-01-15T10:00:10Z", "ip": "203.0.113.30", "method": "GET", "path": "/docs", "status": 404 }, + { "timestamp": "2026-01-15T10:00:11Z", "ip": "192.0.2.12", "method": "GET", "path": "/", "status": 200 } +] diff --git a/examples/components/script-only-pipeline/demo/browser.js b/examples/components/script-only-pipeline/demo/browser.js new file mode 100644 index 000000000..0ccfe46af --- /dev/null +++ b/examples/components/script-only-pipeline/demo/browser.js @@ -0,0 +1,228 @@ +import sample from './access-logs.json'; +import { createPipeline } from './shared/pipeline.js'; +import { readScript } from './shared/read-script.js'; +import { createCustomComponent } from './custom-component.js'; +import { mountEditor } from './editor.js'; +import customTemplate from './custom-template.ts?raw'; + +const $ = (selector) => document.querySelector(selector); +const stages = []; +let pipeline; +let busy = true; +let editorView; +let customDownload; + +function status(message, state = 'ready') { + $('#status').textContent = message; + $('#status').dataset.state = state; +} + +function controls() { + for (const element of document.querySelectorAll('button, input, .stage-options, #source')) { + element.disabled = busy || !pipeline; + } + if (editorView) { + editorView.dom.inert = busy || !pipeline; + } +} + +function invalidate() { + $('#output').textContent = '{}'; + $('#steps').replaceChildren(); + $('#count').textContent = 'No run yet'; + $('#output-note').textContent = 'Run the pipeline to see results for this configuration.'; +} + +function render() { + $('#empty').hidden = stages.length > 0; + $('#pipeline').replaceChildren(); + stages.forEach((stage, index) => { + const row = document.createElement('li'); + const heading = document.createElement('div'); + heading.className = 'stage-heading'; + const title = document.createElement('strong'); + title.textContent = stage.description.name; + const actions = document.createElement('div'); + actions.className = 'stage-actions'; + for (const [label, delta] of [ + ['Move up', -1], + ['Move down', 1], + ['Remove', 0], + ]) { + const button = document.createElement('button'); + button.textContent = label; + button.setAttribute('aria-label', `${label}: ${stage.description.name}`); + button.addEventListener('click', () => { + const target = index + delta; + if (delta && (target < 0 || target >= stages.length)) { + return; + } + if (delta) { + [stages[index], stages[target]] = [stages[target], stages[index]]; + } else { + stages.splice(index, 1); + } + render(); + invalidate(); + status('Pipeline updated · run to refresh results'); + }); + actions.append(button); + } + heading.append(title, actions); + const detail = document.createElement('p'); + detail.className = 'stage-description'; + detail.textContent = `${stage.script.name} · ${stage.description.description}`; + const label = document.createElement('label'); + label.textContent = 'Options (JSON)'; + const options = document.createElement('textarea'); + options.className = 'stage-options'; + options.rows = 2; + options.value = stage.options; + options.spellcheck = false; + options.addEventListener('input', () => { + stage.options = options.value; + invalidate(); + }); + label.append(options); + row.append(heading, detail, label); + $('#pipeline').append(row); + }); + controls(); +} + +$('#components').addEventListener('change', async (event) => { + busy = true; + controls(); + status('Reading components in the browser…', 'loading'); + try { + // Validate the whole selection before changing the pipeline. + const additions = []; + for (const file of event.target.files) { + const script = await readScript(new Uint8Array(await file.arrayBuffer())); + const description = pipeline.describe(script); + additions.push({ script, description, options: description.defaultOptions }); + } + stages.push(...additions); + invalidate(); + status(`Ready · ${stages.length} steps sharing one engine`); + } catch (error) { + status(`Could not add component: ${error.message ?? error}`, 'error'); + } finally { + event.target.value = ''; + busy = false; + render(); + } +}); + +$('#run').addEventListener('click', () => { + invalidate(); + try { + const result = pipeline.run( + $('#input').value, + stages.map((stage) => ({ + script: stage.script, + options: JSON.parse(stage.options), + })), + ); + $('#output').textContent = JSON.stringify({ logs: result.logs, reports: result.reports }, null, 2); + $('#count').textContent = `${result.logs.length} logs / ${stages.length} steps`; + $('#output-note').textContent = 'Every run starts with fresh input. Steps share data inside the engine.'; + for (const step of result.steps) { + const row = document.createElement('li'); + row.textContent = step.summary; + $('#steps').append(row); + } + status('Pipeline complete · shared engine ready for another run'); + } catch (error) { + status(`Pipeline failed: ${error.message ?? error}`, 'error'); + } +}); + +$('#clear').addEventListener('click', () => { + stages.length = 0; + render(); + invalidate(); + status('Pipeline cleared · upload a component to start again'); +}); +$('#restore').addEventListener('click', restoreSample); +$('#input').addEventListener('input', invalidate); +function restoreSample() { + $('#input').value = JSON.stringify(sample, null, 2); + invalidate(); +} +restoreSample(); + +function clearDownload() { + if (customDownload) { + URL.revokeObjectURL(customDownload); + customDownload = undefined; + } + $('#download-custom').hidden = true; + $('#download-custom').removeAttribute('href'); +} + +$('#source').value = customTemplate; +$('#source').addEventListener('input', clearDownload); +$('#component-name').addEventListener('input', clearDownload); +$('#reset-source').addEventListener('click', () => { + $('#source').value = customTemplate; + editorView?.dispatch({ changes: { from: 0, to: editorView.state.doc.length, insert: customTemplate } }); + clearDownload(); +}); +mountEditor($('#source'), $('#source-editor')) + .then((view) => { + editorView = view; + controls(); + $('#editor-status').textContent = + 'TypeScript syntax highlighting by CodeMirror. Editor and compiler load from a CDN.'; + }) + .catch(() => { + $('#editor-status').textContent = + 'Could not load CodeMirror from the CDN. You can still edit the source in the text area.'; + }); + +$('#create-component').addEventListener('click', async () => { + busy = true; + controls(); + clearDownload(); + const name = $('#component-name').value.trim(); + $('#build-status').textContent = 'Bundling TypeScript and creating a script-only Wasm component…'; + $('#build-status').dataset.state = 'building'; + try { + const bytes = await createCustomComponent(name, $('#source').value); + // Take the same Wasm → script → engine path as a file upload. + const script = await readScript(bytes); + const description = pipeline.describe(script); + JSON.parse(description.defaultOptions); + stages.push({ script, description, options: description.defaultOptions }); + customDownload = URL.createObjectURL(new Blob([bytes], { type: 'application/wasm' })); + $('#download-custom').href = customDownload; + $('#download-custom').download = `${name}.wasm`; + $('#download-custom').hidden = false; + $('#build-status').textContent = `Created ${name}.wasm (${bytes.length} bytes) and added it to the pipeline.`; + $('#build-status').dataset.state = 'ready'; + invalidate(); + status(`Ready · ${stages.length} steps sharing one engine`); + } catch (error) { + $('#build-status').textContent = `Could not create component: ${error.message ?? error}`; + $('#build-status').dataset.state = 'error'; + } finally { + busy = false; + render(); + } +}); + +try { + pipeline = await createPipeline(async (url) => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Could not load ${url}: HTTP ${response.status}`); + } + return new Uint8Array(await response.arrayBuffer()); + }); + busy = false; + controls(); + status('Ready · one shared engine, empty pipeline'); +} catch (error) { + status(`Could not start the engine: ${error.message ?? error}`, 'error'); +} diff --git a/examples/components/script-only-pipeline/demo/custom-component.js b/examples/components/script-only-pipeline/demo/custom-component.js new file mode 100644 index 000000000..1a4597871 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/custom-component.js @@ -0,0 +1,70 @@ +import { $init, scripts } from '@bytecodealliance/jco-transpile/component'; +import sharedState from '../components/shared/state.ts?raw'; + +// Pin the JS and Wasm halves to the same version; neither runs on a server. +const ESBUILD_URL = 'https://cdn.jsdelivr.net/npm/esbuild-wasm@0.28.1/esm/browser.min.js'; +const ESBUILD_WASM_URL = 'https://cdn.jsdelivr.net/npm/esbuild-wasm@0.28.1/esbuild.wasm'; +let compiler; + +async function getCompiler() { + compiler ??= (async () => { + const esbuild = await import(/* @vite-ignore */ ESBUILD_URL); + await esbuild.initialize({ wasmURL: ESBUILD_WASM_URL }); + return esbuild; + })().catch((error) => { + compiler = undefined; + throw error; + }); + return compiler; +} + +/** Bundle TypeScript into the loader's ES module shape and package it as real Wasm. */ +export async function createCustomComponent(name, source) { + if (!/^[a-z][a-z0-9-]*$/.test(name) || ['engine', 'dataset'].includes(name)) { + throw new Error('Use a lowercase component name with letters, numbers, and hyphens (except engine or dataset)'); + } + const esbuild = await getCompiler(); + const result = await esbuild.build({ + stdin: { contents: source, sourcefile: `${name}.ts`, loader: 'ts' }, + bundle: true, + write: false, + format: 'esm', + platform: 'neutral', + target: 'es2022', + metafile: true, + logLevel: 'silent', + plugins: [ + { + name: 'pipeline-state', + setup(build) { + build.onResolve({ filter: /.*/ }, ({ path }) => + path === 'pipeline:state' + ? { path, namespace: 'pipeline' } + : { + errors: [ + { + text: 'Runtime imports must use pipeline:state; other modules are not available in the engine', + }, + ], + }, + ); + build.onLoad({ filter: /.*/, namespace: 'pipeline' }, () => ({ + contents: sharedState, + loader: 'ts', + })); + }, + }, + ], + }); + const exports = Object.values(result.metafile.outputs).flatMap((output) => output.exports); + if (exports.length !== 1 || exports[0] !== 'processor') { + throw new Error('Export one processor object with describe() and process(options) methods'); + } + await $init; + return scripts.createComponent({ + name: `${name}.js`, + source: result.outputFiles[0].text, + engineWorld: 'jco:script-engine/engine@0.1.0', + exports, + }); +} diff --git a/examples/components/script-only-pipeline/demo/custom-template.ts b/examples/components/script-only-pipeline/demo/custom-template.ts new file mode 100644 index 000000000..4a6497d96 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/custom-template.ts @@ -0,0 +1,21 @@ +import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; +import { getDataset, parseOptions } from 'pipeline:state'; + +export const processor: typeof ProcessorInterface = { + describe() { + return { + name: 'HTTP status filter', + description: 'Keep requests with an HTTP status at or above a threshold.', + defaultOptions: JSON.stringify({ minStatus: 400 }), + }; + }, + process(json) { + const { minStatus = 400 } = parseOptions(json); + if (typeof minStatus !== 'number' || minStatus < 100 || minStatus > 599) { + throw 'minStatus must be a number from 100 to 599'; + } + const data = getDataset(); + data.logs = data.logs.filter((log) => log.status >= minStatus); + return `HTTP status: kept ${data.logs.length} logs with status >= ${minStatus}`; + }, +}; diff --git a/examples/components/script-only-pipeline/demo/editor.js b/examples/components/script-only-pipeline/demo/editor.js new file mode 100644 index 000000000..a0eb0c8df --- /dev/null +++ b/examples/components/script-only-pipeline/demo/editor.js @@ -0,0 +1,28 @@ +const CODEMIRROR_URL = 'https://esm.sh/codemirror@6.0.2'; +const JAVASCRIPT_URL = 'https://esm.sh/@codemirror/lang-javascript@6.2.4'; + +/** Enhance the source textarea without making the rest of the demo depend on a CDN. */ +export async function mountEditor(textarea, parent) { + const [{ EditorView, basicSetup }, { javascript }] = await Promise.all([ + import(/* @vite-ignore */ CODEMIRROR_URL), + import(/* @vite-ignore */ JAVASCRIPT_URL), + ]); + const view = new EditorView({ + parent, + doc: textarea.value, + extensions: [ + basicSetup, + javascript({ typescript: true }), + EditorView.contentAttributes.of({ 'aria-label': 'Custom processor TypeScript source' }), + EditorView.updateListener.of((update) => { + if (update.docChanged) { + textarea.value = update.state.doc.toString(); + textarea.dispatchEvent(new Event('input', { bubbles: true })); + } + }), + ], + }); + textarea.hidden = true; + parent.dataset.ready = 'true'; + return view; +} diff --git a/examples/components/script-only-pipeline/demo/index.html b/examples/components/script-only-pipeline/demo/index.html new file mode 100644 index 000000000..1a713f9c3 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/index.html @@ -0,0 +1,91 @@ + + + + + + Access log lab · Jco script-only components + + + + + +
+
+

JCO / SCRIPT-ONLY COMPONENTS

+

Access log lab

+

This example showcases WebAssembly components used in a dynamic pipeline.

+

The controls on this page enable processing one shared dataset with multiple scripts compiled to WebAssembly and one shared JavaScript engine.

+

Starting the shared engine…

+
+
+
+

Your pipeline

+ +
+

Download the sample components, then upload them in the order you want to run them.

+ + +

Your pipeline is empty. Upload a component to add the first step.

+
    +

    + Use trusted scripts: components loaded into this engine share a JavaScript realm and dataset. +

    +
    +
    +
    +

    Write your own component

    + +
    +

    Write a TypeScript processor, create its .wasm component here, and add it to your pipeline.

    +

    + Export a processor with describe() and process(options). Import getDataset and parseOptions from + pipeline:state to work with the shared logs and reports. No server build is needed. +

    + + + + +
    +

    Loading CodeMirror syntax highlighting…

    +
    + + +
    +

    + Your component will be built locally in this browser. +

    +
    +
    +
    +
    +

    01 / Input

    + +
    + + +

    + Synthetic logs and invented locations for documentation IP ranges. Risk scores are illustrative. +

    + +
    +
    +
    +

    02 / Output

    + No run yet +
    +

    Run the empty pipeline to see the original logs, or add a few steps first.

    +
      + 
      {}
      +
      +
      +
      + + diff --git a/examples/components/script-only-pipeline/demo/node.mjs b/examples/components/script-only-pipeline/demo/node.mjs new file mode 100644 index 000000000..28f907910 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/node.mjs @@ -0,0 +1,41 @@ +import { readFile, readdir } from 'node:fs/promises'; +import { createPipeline } from './shared/pipeline.js'; +import { readScript } from './shared/read-script.js'; + +export async function main(args = process.argv.slice(2)) { + // CLI arguments take precedence over the comma-separated environment setting. + // An explicitly empty PIPELINE_STAGES runs the input through an empty pipeline. + const names = (args.length ? args : (process.env.PIPELINE_STAGES ?? 'geoip,risk-score,top-pages').split(',')) + .map((name) => name.trim()) + .filter(Boolean); + const components = new URL('../dist/components/', import.meta.url); + const available = (await readdir(components)) + .filter((file) => file.endsWith('.wasm') && !['engine.wasm', 'dataset.wasm'].includes(file)) + .map((file) => file.slice(0, -'.wasm'.length)); + for (const name of names) { + if (!available.includes(name)) { + throw new Error(`Unknown stage "${name}". Available stages: ${available.join(', ')}`); + } + } + + const pipeline = await createPipeline(readFile); + const input = await readFile(new URL('./access-logs.json', import.meta.url), 'utf8'); + const stages = []; + for (const name of names) { + const script = await readScript(await readFile(new URL(`${name}.wasm`, components))); + const description = pipeline.describe(script); + stages.push({ script, options: JSON.parse(description.defaultOptions) }); + } + const result = pipeline.run(input, stages); + for (const step of result.steps) { + console.log(step.summary); + } + console.log(JSON.stringify({ logs: result.logs, reports: result.reports }, null, 2)); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error.message ?? error); + process.exitCode = 1; + }); +} diff --git a/examples/components/script-only-pipeline/demo/shared/pipeline.js b/examples/components/script-only-pipeline/demo/shared/pipeline.js new file mode 100644 index 000000000..538be0bee --- /dev/null +++ b/examples/components/script-only-pipeline/demo/shared/pipeline.js @@ -0,0 +1,47 @@ +import { instantiate as instantiateEngine } from '../../dist/transpiled/engine/engine.js'; +import { instantiate as instantiateDataset } from '../../dist/transpiled/dataset/dataset.js'; + +/** Only byte loading differs between hosts (readFile in Node, fetch in the browser). */ +export async function createPipeline(loadBytes) { + const compile = (directory) => (name) => + loadBytes(new URL(`${directory}/${name}`, import.meta.url)).then((bytes) => WebAssembly.compile(bytes)); + const engine = await instantiateEngine(compile('../../dist/transpiled/engine'), {}); + const { script: dataset } = await instantiateDataset(compile('../../dist/transpiled/dataset'), {}); + const loadDataset = () => engine.scriptLoader.load(dataset.name(), dataset.source()); + const load = (script) => engine.scriptLoader.load(script.name, script.source); + const snapshot = () => { + loadDataset(); + return JSON.parse(engine.dataset.snapshot()); + }; + + return { + describe(script) { + if (script.engineWorld !== dataset.engineWorld() || !script.exports.includes('processor')) { + throw new Error('This pipeline needs a script-only component with a processor export'); + } + try { + load(script); + return engine.processor.describe(); + } finally { + engine.scriptLoader.unload(); + } + }, + run(input, stages) { + try { + loadDataset(); + engine.dataset.reset(input); + const steps = []; + for (const { script, options } of stages) { + load(script); + const summary = engine.processor.process(JSON.stringify(options)); + // Switching back to the dataset script does not lose the data. + // Only snapshots cross the host boundary; the working set stays in the engine. + steps.push({ name: script.name, summary, ...snapshot() }); + } + return { ...snapshot(), steps }; + } finally { + engine.scriptLoader.unload(); + } + }, + }; +} diff --git a/examples/components/script-only-pipeline/demo/shared/read-script.js b/examples/components/script-only-pipeline/demo/shared/read-script.js new file mode 100644 index 000000000..d0df36843 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/shared/read-script.js @@ -0,0 +1,27 @@ +import { generate } from '@bytecodealliance/jco/component'; + +/** Turn an uploaded script-only component into source and metadata, on either host. */ +export async function readScript(bytes) { + const output = await generate(bytes, { + name: 'uploaded', + instantiation: { tag: 'async' }, + noTypescript: true, + noNodejsCompat: true, + base64Cutoff: 0, + }); + if (output.imports.length || !output.exports.some(([name]) => name === 'jco:script/script@0.1.0')) { + throw new Error('Upload a script-only component exporting jco:script/script@0.1.0'); + } + const files = new Map(output.files); + // Instantiation bindings have no static imports. Supplying the core modules + // explicitly lets the same data URL work in Node.js and the browser. + const source = new TextDecoder().decode(files.get('uploaded.js')); + const { instantiate } = await import(/* @vite-ignore */ `data:text/javascript,${encodeURIComponent(source)}`); + const { script } = await instantiate((name) => WebAssembly.compile(files.get(name)), {}); + return { + name: script.name(), + source: script.source(), + exports: script.exports(), + engineWorld: script.engineWorld(), + }; +} diff --git a/examples/components/script-only-pipeline/demo/style.css b/examples/components/script-only-pipeline/demo/style.css new file mode 100644 index 000000000..8c6c91408 --- /dev/null +++ b/examples/components/script-only-pipeline/demo/style.css @@ -0,0 +1,233 @@ +:root { + font-family: system-ui, sans-serif; + color: #182d36; + background: #f0f4f2; + line-height: 1.6; + color-scheme: light; +} +* { + box-sizing: border-box; +} +[hidden] { + display: none !important; +} +body { + margin: 0; +} +main { + max-width: 1240px; + margin: auto; + padding: 48px 24px; +} +header { + max-width: 780px; + margin-bottom: 32px; +} +h1 { + font-size: clamp(2.5rem, 6vw, 4rem); + letter-spacing: -0.055em; + line-height: 1.1; + margin: 12px 0; +} +h2 { + font-size: 1.1rem; + margin: 0; +} +p { + margin: 12px 0; +} +.eyebrow { + font-size: 0.75rem; + letter-spacing: 0.16em; + font-weight: 700; + color: #256657; +} +section { + background: #fff; + border: 1px solid #ccd9d2; + border-radius: 12px; + padding: 24px; + min-width: 0; +} +.section-heading, +.stage-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} +.downloads { + display: flex; + flex-wrap: wrap; + gap: 12px; + margin: 16px 0; +} +a { + color: #18634e; + font-weight: 600; +} +.upload { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 12px; + padding: 20px; + border: 2px dashed #9ab8ab; + border-radius: 8px; + background: #f7faf8; +} +.upload span, +.note { + font-size: 0.85rem; + color: #526960; +} +.upload input { + max-width: 100%; +} +.columns { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 20px; + margin-top: 20px; +} +button { + cursor: pointer; + border: 1px solid #a9bfb4; + border-radius: 6px; + padding: 8px 12px; + background: #fff; + color: #1a5543; + font: inherit; + font-size: 0.85rem; +} +button:hover { + background: #eaf3ed; +} +button:disabled { + opacity: 0.45; + cursor: default; +} +button.primary { + background: #195f4d; + color: white; + padding: 12px 24px; + font-weight: 600; +} +:focus-visible { + outline: 3px solid #d89637; + outline-offset: 3px; +} +textarea, +pre { + width: 100%; + padding: 16px; + border-radius: 6px; + font: + 0.8rem/1.6 ui-monospace, + monospace; + tab-size: 2; +} +textarea { + display: block; + background: #f9fbfa; + border: 1px solid #c6d6cd; + margin-top: 8px; + resize: vertical; +} +pre { + background: #172e29; + color: #e0f0e7; + overflow: auto; + max-height: 600px; + margin-bottom: 0; +} +#status { + padding: 10px 14px; + border-left: 3px solid #34806a; + background: #e2eee7; +} +#status[data-state='error'] { + border-color: #b84434; + color: #8f291c; + background: #ffede7; +} +#pipeline { + padding-left: 24px; +} +#pipeline > li { + padding: 16px 0; + border-bottom: 1px solid #dbe5df; +} +.stage-actions { + display: flex; + gap: 6px; +} +.stage-options { + min-height: 70px; +} +.stage-description { + font-size: 0.9rem; + color: #526960; +} +#empty { + text-align: center; + padding: 24px; + color: #526960; +} +#count { + font-size: 0.85rem; + color: #526960; +} +#authoring { + margin-top: 20px; +} +#authoring > label { + display: block; + margin-top: 12px; +} +#component-name { + font: inherit; + padding: 8px 12px; + max-width: 100%; + border: 1px solid #c6d6cd; + border-radius: 6px; +} +#source-editor { + min-width: 0; + margin-top: 8px; +} +#source-editor .cm-editor { + border: 1px solid #c6d6cd; + border-radius: 6px; + overflow: hidden; + font-size: 0.85rem; +} +#source-editor .cm-scroller { + max-height: 480px; + overflow: auto; +} +.authoring-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 16px; + margin-top: 16px; +} +#build-status[data-state='error'] { + color: #8f291c; +} +@media (max-width: 760px) { + main { + padding: 24px 14px; + } + .columns { + grid-template-columns: 1fr; + } + section { + padding: 18px; + } + .stage-heading { + align-items: flex-start; + flex-direction: column; + } +} diff --git a/examples/components/script-only-pipeline/demo/vite.config.js b/examples/components/script-only-pipeline/demo/vite.config.js new file mode 100644 index 000000000..32030d12a --- /dev/null +++ b/examples/components/script-only-pipeline/demo/vite.config.js @@ -0,0 +1,19 @@ +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; +import { defineConfig, searchForWorkspaceRoot } from 'vite'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const jcoRequire = createRequire(import.meta.resolve('@bytecodealliance/jco/component')); +const bindgenDirectory = dirname(jcoRequire.resolve('@bytecodealliance/jco-transpile/component')); + +export default defineConfig({ + root, + // Jco's browser entry loads its own Wasm with import.meta.url. Leave those + // URLs intact instead of relocating the module into Vite's dependency cache. + optimizeDeps: { exclude: ['@bytecodealliance/jco', '@bytecodealliance/jco-transpile'] }, + server: { + // Also serve bindgen's Wasm assets when Jco is linked from another checkout. + fs: { allow: [searchForWorkspaceRoot(root), bindgenDirectory] }, + }, +}); diff --git a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts new file mode 100644 index 000000000..9cda9fb0a --- /dev/null +++ b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts @@ -0,0 +1,19 @@ +declare module 'example:log-pipeline/dataset' { + /** + * Replace the shared dataset with a JSON array of access-log records. + */ + export function reset(json: string): void; + /** + * Read the current logs and reports as JSON without resetting them. + */ + export function snapshot(): string; + export interface AccessLog { + timestamp: string, + ip: string, + method: string, + path: string, + status: number, + country?: string, + riskScore?: number, + } +} diff --git a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts new file mode 100644 index 000000000..ec1f57202 --- /dev/null +++ b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts @@ -0,0 +1,12 @@ +declare module 'example:log-pipeline/processor' { + export function describe(): Description; + /** + * Mutate the shared dataset and return a short summary, or an error. + */ + export function process(options: string): string; + export interface Description { + name: string, + description: string, + defaultOptions: string, + } +} diff --git a/examples/components/script-only-pipeline/generated/types/pipeline.d.ts b/examples/components/script-only-pipeline/generated/types/pipeline.d.ts new file mode 100644 index 000000000..c98696d19 --- /dev/null +++ b/examples/components/script-only-pipeline/generated/types/pipeline.d.ts @@ -0,0 +1,6 @@ +/// +/// +declare module 'example:log-pipeline/pipeline' { + export * as dataset from 'example:log-pipeline/dataset'; // export example:log-pipeline/dataset + export * as processor from 'example:log-pipeline/processor'; // export example:log-pipeline/processor +} diff --git a/examples/components/script-only-pipeline/package.json b/examples/components/script-only-pipeline/package.json new file mode 100644 index 000000000..e934d6b45 --- /dev/null +++ b/examples/components/script-only-pipeline/package.json @@ -0,0 +1,30 @@ +{ + "name": "script-only-pipeline", + "version": "0.1.0", + "private": true, + "description": "Process JSON access logs with script-only components sharing one engine in Node.js and the browser", + "type": "module", + "scripts": { + "gen:types": "jco guest-types wit --world-name pipeline -o generated/types", + "check": "tsc --noEmit", + "build:component": "jco componentize --script-only --backend quickjs --backend-qjs-disable-async --backend-qjs-stub-wasi --wit wit --world-name pipeline components/dataset/dataset.ts components/geoip/geoip.ts components/risk-score/risk-score.ts components/top-pages/top-pages.ts -o dist/components", + "transpile": "node scripts/transpile.mjs", + "build": "pnpm run gen:types && pnpm run check && pnpm run build:component && pnpm run transpile", + "run": "node demo/node.mjs", + "demo": "vite --config demo/vite.config.js", + "demo:open": "pnpm run demo --open /demo/", + "test:node": "node test/pipeline.mjs", + "test:browser": "node test/browser.mjs", + "test": "pnpm run test:node && pnpm run test:browser", + "all": "pnpm run build && pnpm run run && pnpm run test" + }, + "dependencies": { + "@bytecodealliance/jco": "workspace:*", + "@bytecodealliance/jco-transpile": "^0.21.0" + }, + "devDependencies": { + "puppeteer": "catalog:", + "typescript": "catalog:", + "vite": "^7.3.7" + } +} diff --git a/examples/components/script-only-pipeline/scripts/transpile.mjs b/examples/components/script-only-pipeline/scripts/transpile.mjs new file mode 100644 index 000000000..1d8547311 --- /dev/null +++ b/examples/components/script-only-pipeline/scripts/transpile.mjs @@ -0,0 +1,20 @@ +import { execFileSync } from 'node:child_process'; + +// The engine and data initializer are known at build time. Processing scripts are +// deliberately left as .wasm files so both hosts can load them at runtime. +for (const name of ['engine', 'dataset']) { + execFileSync( + 'jco', + [ + 'transpile', + `dist/components/${name}.wasm`, + '--name', + name, + '--instantiation', + 'async', + '-o', + `dist/transpiled/${name}`, + ], + { stdio: 'inherit' }, + ); +} diff --git a/examples/components/script-only-pipeline/test/browser.mjs b/examples/components/script-only-pipeline/test/browser.mjs new file mode 100644 index 000000000..8a075798e --- /dev/null +++ b/examples/components/script-only-pipeline/test/browser.mjs @@ -0,0 +1,214 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import puppeteer from 'puppeteer'; +import { createServer } from 'vite'; +import config from '../demo/vite.config.js'; +import { createPipeline } from '../demo/shared/pipeline.js'; +import { readScript } from '../demo/shared/read-script.js'; + +const component = (name) => fileURLToPath(new URL(`../dist/components/${name}.wasm`, import.meta.url)); +const input = await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8'); +const nodePipeline = await createPipeline(readFile); +const nodeStages = []; +for (const name of ['geoip', 'risk-score', 'top-pages']) { + const script = await readScript(await readFile(component(name))); + nodeStages.push({ script, options: JSON.parse(nodePipeline.describe(script).defaultOptions) }); +} +const expected = nodePipeline.run(input, nodeStages); +const server = await createServer({ + ...config, + configFile: false, + logLevel: 'silent', + server: { ...config.server, host: '127.0.0.1', port: 0, open: false }, +}); +let browser; +const diagnostics = []; +const downloads = await mkdtemp(join(tmpdir(), 'jco-pipeline-download-')); +try { + await server.listen(); + browser = await puppeteer.launch({ + executablePath: process.env.PUPPETEER_PATH || undefined, + args: (process.env.TEST_PUPPETEER_LAUNCH_ARGS ?? '').split(/[\s,]+/).filter(Boolean), + }); + const page = await browser.newPage(); + page.on('pageerror', (error) => diagnostics.push(`pageerror: ${error.message}`)); + page.on('console', (message) => { + if (message.type() === 'error') { + diagnostics.push(`console: ${message.text()}`); + } + }); + page.on('requestfailed', (request) => diagnostics.push(`requestfailed: ${request.url()}`)); + page.on('response', (response) => { + if (response.status() >= 400) { + diagnostics.push(`HTTP ${response.status()}: ${response.url()}`); + } + }); + const response = await page.goto(`http://127.0.0.1:${server.httpServer.address().port}/demo/`); + assert(response.ok()); + await page.waitForFunction(() => ['ready', 'error'].includes(document.querySelector('#status').dataset.state), { + timeout: 60_000, + }); + assert.equal( + await page.$eval('#status', (el) => el.dataset.state), + 'ready', + await page.$eval('#status', (el) => el.textContent), + ); + assert.equal((await page.$$('#pipeline > li')).length, 0); + assert.equal(await page.$eval('#empty', (el) => el.hidden), false); + await page.click('#run'); + assert.deepEqual(await output(page), { logs: JSON.parse(input), reports: {} }); + + // Download links serve actual component binaries, ready to upload again. + for (const name of ['geoip', 'risk-score', 'top-pages']) { + const bytes = await page.evaluate( + async (name) => [...new Uint8Array(await (await fetch(`../dist/components/${name}.wasm`)).arrayBuffer())], + name, + ); + assert.deepEqual(new Uint8Array(bytes), new Uint8Array(await readFile(component(name)))); + } + + await upload(page, component('dataset')); + assert.match(await page.$eval('#status', (el) => el.textContent), /processor export/); + assert.equal((await page.$$('#pipeline > li')).length, 0); + + await upload(page, ...['geoip', 'risk-score', 'top-pages'].map(component)); + assert.equal((await page.$$('#pipeline > li')).length, 3); + await page.click('#run'); + assert.deepEqual(await output(page), { logs: expected.logs, reports: expected.reports }); + assert.equal((await page.$$('#steps > li')).length, 3); + + await setValue(page, '#pipeline > li:nth-child(2) textarea', '{"minScore":70}'); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 3); + await page.click('button[aria-label="Move up: Most visited pages"]'); + await page.click('button[aria-label="Move up: Most visited pages"]'); + await page.click('#run'); + assert.equal((await output(page)).reports.topPages[0].visits, 4); + await page.click('button[aria-label="Remove: Risk score"]'); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 5); + + await upload(page, fileURLToPath(new URL('../demo/access-logs.json', import.meta.url))); + assert.equal(await page.$eval('#status', (el) => el.dataset.state), 'error'); + assert.equal((await page.$$('#pipeline > li')).length, 2); + await setValue(page, '#pipeline > li:first-child textarea', '{"limit":0}'); + await page.click('#run'); + assert.equal(await page.$eval('#status', (el) => el.dataset.state), 'error'); + assert.deepEqual(await output(page), {}); + await setValue(page, '#pipeline > li:first-child textarea', '{"limit":5}'); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 5); + + await page.click('#clear'); + await setValue(page, '#input', '{broken'); + await page.click('#run'); + assert.equal(await page.$eval('#status', (el) => el.dataset.state), 'error'); + await page.click('#restore'); + await page.click('#run'); + assert.deepEqual(await output(page), { logs: JSON.parse(input), reports: {} }); + assert.equal((await page.$$('#pipeline > li')).length, 0); + await page.waitForFunction(() => document.querySelector('#source-editor').dataset.ready === 'true', { + timeout: 60_000, + }); + assert( + (await page.$$('#source-editor .cm-content span')).length > 0, + 'CodeMirror should highlight the TypeScript source', + ); + + await createComponent(page); + assert.equal((await page.$$('#pipeline > li')).length, 1); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 4); + + // Edit the actual highlighted editor, including a TypeScript type annotation. + const customSource = (await readFile(new URL('../demo/custom-template.ts', import.meta.url), 'utf8')).replace( + 'minStatus: 400', + 'minStatus: 500', + ); + await editSource(page, customSource); + await createComponent(page); + assert.equal((await page.$$('#pipeline > li')).length, 2); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 1); + + // The downloadable bytes are a real script component, reusable by either host. + assert.equal(await page.$eval('#download-custom', (el) => el.download), 'http-status.wasm'); + const bytes = new Uint8Array( + await page.$eval('#download-custom', async (el) => [ + ...new Uint8Array(await (await fetch(el.href)).arrayBuffer()), + ]), + ); + const customScript = await readScript(bytes); + const description = nodePipeline.describe(customScript); + const fromNode = nodePipeline.run(input, [ + { script: customScript, options: JSON.parse(description.defaultOptions) }, + ]); + assert.deepEqual(await output(page), { logs: fromNode.logs, reports: fromNode.reports }); + const downloaded = join(downloads, 'http-status.wasm'); + await writeFile(downloaded, bytes); + await page.click('#clear'); + await upload(page, downloaded); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 1); + + await editSource(page, 'export const processor = {'); + await createComponent(page, 'error'); + assert.equal((await page.$$('#pipeline > li')).length, 1); + assert.equal(await page.$eval('#download-custom', (el) => el.hidden), true); + await page.click('#reset-source'); + await createComponent(page); + await page.click('#run'); + assert.equal((await output(page)).logs.length, 1); + + assert.equal(diagnostics.filter((entry) => entry.startsWith('pageerror:')).length, 0, diagnostics.join('\n')); + console.log('Browser uploads, custom TypeScript builds, download round trips, recovery, and Node.js parity passed'); +} catch (error) { + console.error(diagnostics.join('\n')); + throw error; +} finally { + await browser?.close(); + await server.close(); + await rm(downloads, { recursive: true, force: true }); +} + +async function upload(page, ...paths) { + await (await page.$('#components')).uploadFile(...paths); + await page.waitForFunction(() => !document.querySelector('#components').disabled); +} + +async function setValue(page, selector, value) { + await page.$eval( + selector, + (element, value) => { + element.value = value; + element.dispatchEvent(new Event('input', { bubbles: true })); + }, + value, + ); +} + +async function output(page) { + return JSON.parse(await page.$eval('#output', (element) => element.textContent)); +} + +async function createComponent(page, expectedState = 'ready') { + await page.click('#create-component'); + await page.waitForFunction(() => !document.querySelector('#create-component').disabled, { timeout: 120_000 }); + assert.equal( + await page.$eval('#build-status', (el) => el.dataset.state), + expectedState, + await page.$eval('#build-status', (el) => el.textContent), + ); +} + +async function editSource(page, source) { + await page.click('#source-editor .cm-content'); + await page.keyboard.down('Control'); + await page.keyboard.press('KeyA'); + await page.keyboard.up('Control'); + await page.keyboard.insertText(source); + await page.waitForFunction((source) => document.querySelector('#source').value === source, {}, source); +} diff --git a/examples/components/script-only-pipeline/test/pipeline.mjs b/examples/components/script-only-pipeline/test/pipeline.mjs new file mode 100644 index 000000000..760eb0715 --- /dev/null +++ b/examples/components/script-only-pipeline/test/pipeline.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { createPipeline } from '../demo/shared/pipeline.js'; +import { readScript } from '../demo/shared/read-script.js'; + +const pipeline = await createPipeline(readFile); +const input = await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8'); +const scripts = {}; +const stages = []; +for (const name of ['geoip', 'risk-score', 'top-pages']) { + const script = await readScript(await component(name)); + scripts[name] = script; + stages.push({ script, options: JSON.parse(pipeline.describe(script).defaultOptions) }); +} + +function component(name) { + return readFile(new URL(`../dist/components/${name}.wasm`, import.meta.url)); +} + +test('an empty pipeline returns the original logs', () => { + assert.deepEqual(pipeline.run(input, []), { logs: JSON.parse(input), reports: {}, steps: [] }); +}); + +test('three separately packaged scripts share annotations and reports in one engine', () => { + const result = pipeline.run(input, stages); + assert.equal(result.steps.length, 3); + assert.equal(result.logs.length, 5); + assert(result.logs.every((log) => log.country === 'DE')); + assert.deepEqual( + result.logs.map((log) => log.riskScore), + [0, 70, 70, 0, 90], + ); + assert.deepEqual(result.reports.topPages, [ + { path: '/docs', visits: 2 }, + { path: '/login', visits: 2 }, + { path: '/admin', visits: 1 }, + ]); + // The first step's snapshot predates scoring; snapshots are independent copies. + assert(result.steps[0].logs.every((log) => log.riskScore === undefined)); + assert.deepEqual(pipeline.run(input, stages), result); +}); + +test('options and order change the results', () => { + const risky = pipeline.run(input, [ + stages[0], + { script: scripts['risk-score'], options: { minScore: 70 } }, + stages[2], + ]); + assert.equal(risky.logs.length, 3); + assert.deepEqual(risky.reports.topPages, [ + { path: '/login', visits: 2 }, + { path: '/admin', visits: 1 }, + ]); + const countFirst = pipeline.run(input, [stages[2], stages[0]]); + assert.equal(countFirst.logs.length, 5); + assert.equal(countFirst.reports.topPages[0].visits, 4); + assert.equal( + countFirst.reports.topPages.reduce((sum, page) => sum + page.visits, 0), + 12, + ); +}); + +test('empty datasets and a new run cannot inherit previous annotations or reports', () => { + pipeline.run(input, stages); + assert.deepEqual(pipeline.run('[]', stages).reports, { topPages: [] }); + assert.deepEqual(pipeline.run(input, []), { logs: JSON.parse(input), reports: {}, steps: [] }); +}); + +test('invalid input and options produce recoverable component errors', () => { + for (const invalid of ['{', '{}', '[{}]', '[{"status":999}]']) { + assert.throws(() => pipeline.run(invalid, stages)); + } + for (const [name, options] of [ + ['geoip', { country: 'XX' }], + ['geoip', { country: ['DE'] }], + ['risk-score', { minScore: -1 }], + ['top-pages', { limit: 0 }], + ['top-pages', []], + ]) { + assert.throws(() => pipeline.run(input, [{ script: scripts[name], options }])); + } + assert.equal(pipeline.run(input, stages).logs.length, 5); +}); + +test('uploads must be processor script-only components', async () => { + await assert.rejects(readScript(new TextEncoder().encode('not wasm'))); + await assert.rejects(readScript(await component('engine')), /script-only component/); + const dataset = await readScript(await component('dataset')); + assert.throws(() => pipeline.describe(dataset), /processor export/); + assert.throws(() => pipeline.describe({ ...scripts.geoip, engineWorld: 'wrong:world/world' }), /processor export/); +}); + +test('different engine instances have separate datasets', async () => { + const other = await createPipeline(readFile); + assert.equal(other.run('[]', stages).logs.length, 0); + assert.equal(pipeline.run(input, stages).logs.length, 5); + assert.equal(other.run(input, []).logs.length, 12); +}); + +const exec = promisify(execFile); +const demoUrl = new URL('../demo/node.mjs', import.meta.url); +const demoEnv = { ...process.env }; +delete demoEnv.PIPELINE_STAGES; + +async function runDemo(args = [], env = {}) { + const { stdout } = await exec(process.execPath, [fileURLToPath(demoUrl), ...args], { + env: { ...demoEnv, ...env }, + }); + return { + summaries: stdout.slice(0, stdout.indexOf('{')).trim(), + result: JSON.parse(stdout.slice(stdout.indexOf('{'))), + }; +} + +test('the Node CLI selects stages from arguments or ENV, with arguments taking precedence', async () => { + const ordered = await runDemo(['top-pages', 'geoip']); + assert.match(ordered.summaries, /^Pages:.*\nGeoIP:/); + assert.equal(ordered.result.logs.length, 5); + assert.equal(ordered.result.reports.topPages[0].visits, 4); + + const fromEnv = await runDemo([], { PIPELINE_STAGES: 'geoip, top-pages' }); + assert.equal(fromEnv.result.logs.length, 5); + assert.equal(fromEnv.result.reports.topPages[0].visits, 2); + + const overridden = await runDemo(['risk-score'], { PIPELINE_STAGES: 'geoip' }); + assert.equal(overridden.result.logs.length, 12); + assert(overridden.result.logs.every((log) => typeof log.riskScore === 'number' && log.country === undefined)); + + const empty = await runDemo([], { PIPELINE_STAGES: '' }); + assert.equal(empty.summaries, ''); + assert.deepEqual(empty.result, { logs: JSON.parse(input), reports: {} }); +}); + +test('the Node CLI reports unknown stages and exits unsuccessfully', async () => { + await assert.rejects(runDemo(['missing']), (error) => { + assert.equal(error.code, 1); + assert.match(error.stderr, /Unknown stage "missing"\. Available stages:/); + return true; + }); +}); + +test('importing the Node entry point does not run the demo', async () => { + const { stdout, stderr } = await exec(process.execPath, [ + '--input-type=module', + '--eval', + `await import(${JSON.stringify(demoUrl.href)})`, + ]); + assert.equal(stdout, ''); + assert.equal(stderr, ''); +}); diff --git a/examples/components/script-only-pipeline/tsconfig.json b/examples/components/script-only-pipeline/tsconfig.json new file mode 100644 index 000000000..ca647736b --- /dev/null +++ b/examples/components/script-only-pipeline/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "noEmit": true, + "types": [], + "paths": { + "pipeline:state": ["./components/shared/state.ts"] + } + }, + "include": ["components", "generated/types", "demo/custom-template.ts"] +} diff --git a/examples/components/script-only-pipeline/wit/pipeline.wit b/examples/components/script-only-pipeline/wit/pipeline.wit new file mode 100644 index 000000000..c31259ce9 --- /dev/null +++ b/examples/components/script-only-pipeline/wit/pipeline.wit @@ -0,0 +1,35 @@ +package example:log-pipeline; + +interface dataset { + record access-log { + timestamp: string, + ip: string, + method: string, + path: string, + status: u16, + country: option, + risk-score: option, + } + + /// Replace the shared dataset with a JSON array of access-log records. + reset: func(json: string) -> result<_, string>; + /// Read the current logs and reports as JSON without resetting them. + snapshot: func() -> string; +} + +interface processor { + record description { + name: string, + description: string, + default-options: string, + } + + describe: func() -> description; + /// Mutate the shared dataset and return a short summary, or an error. + process: func(options: string) -> result; +} + +world pipeline { + export dataset; + export processor; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa0fab5cd..e36b86a22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -334,6 +334,25 @@ importers: specifier: workspace:* version: link:../../../packages/jco + examples/components/script-only-pipeline: + dependencies: + '@bytecodealliance/jco': + specifier: workspace:* + version: link:../../../packages/jco + '@bytecodealliance/jco-transpile': + specifier: ^0.21.0 + version: 0.21.0 + devDependencies: + puppeteer: + specifier: 'catalog:' + version: 25.14.0(yauzl@2.10.0) + typescript: + specifier: 'catalog:' + version: 7.0.2 + vite: + specifier: ^7.3.7 + version: 7.3.7(@types/node@24.13.3)(jiti@2.6.1)(lightningcss@1.33.0) + examples/components/string-reverse: dependencies: '@bytecodealliance/jco': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 56c03c082..8599d7a5c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -47,6 +47,7 @@ packages: - examples/components/node-fetch - examples/components/perry-filesystem - examples/components/p3-stream-echo-qjs + - examples/components/script-only-pipeline - examples/components/string-reverse - examples/components/string-reverse-upper - examples/components/ts-resource-export From 2f199b515483e362140c340019ceea560e111314 Mon Sep 17 00:00:00 2001 From: Victor Adossi Date: Thu, 29 Oct 2026 16:01:58 +0000 Subject: [PATCH 2/2] fix(examples): use jsDelivr and typed pipeline records --- .../components/script-only-pipeline/README.md | 64 +++++++++---- .../components/dataset/dataset.ts | 28 ++---- .../components/geoip/geoip.ts | 9 +- .../components/risk-score/risk-score.ts | 9 +- .../components/shared/state.ts | 2 +- .../components/top-pages/top-pages.ts | 9 +- .../script-only-pipeline/demo/browser.js | 21 +++-- .../demo/custom-component.js | 2 +- .../demo/custom-template.ts | 9 +- .../script-only-pipeline/demo/editor.js | 11 +-- .../script-only-pipeline/demo/index.html | 33 ++++++- .../script-only-pipeline/demo/node.mjs | 27 +++--- .../demo/shared/pipeline.js | 89 +++++++++++++++---- .../example-log-pipeline-dataset.d.ts | 24 ++--- .../example-log-pipeline-processor.d.ts | 21 ++++- .../example-log-pipeline-types.d.ts | 11 +++ .../generated/types/pipeline.d.ts | 2 + .../script-only-pipeline/test/browser.mjs | 25 ++++-- .../script-only-pipeline/test/pipeline.mjs | 75 ++++++++++++---- .../script-only-pipeline/wit/pipeline.wit | 40 +++++++-- 20 files changed, 372 insertions(+), 139 deletions(-) create mode 100644 examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-types.d.ts diff --git a/examples/components/script-only-pipeline/README.md b/examples/components/script-only-pipeline/README.md index 2aa56e296..0c7f67609 100644 --- a/examples/components/script-only-pipeline/README.md +++ b/examples/components/script-only-pipeline/README.md @@ -28,7 +28,7 @@ Use Node.js 24 or newer. This example uses the workspace version of Jco, which i `pnpm run all` follows the other examples: generate types, check TypeScript, build the components, transpile the engine and dataset initializer, run the Node.js demo, and run Node.js and Puppeteer tests. The QuickJS engine uses a synchronous ABI and stubbed WASI because these processors only operate on -in-memory JSON. Neither host needs JSPI flags, filesystem access from the guest, or WASI shims. +in-memory records. Neither host needs JSPI flags, filesystem access from the guest, or WASI shims. ## Try the browser demo @@ -42,6 +42,10 @@ Open `http://localhost:5173/demo/` after `pnpm run demo`, or use `pnpm run demo: 5. Set the risk step's options to `{"minScore":70}` and rerun. Three logs remain, with scores 70, 70, and 90. 6. Move the page counter to the beginning. Its report now counts all 12 input logs, before filtering. +Choose **Use one sample log** to call `process-single` with one `access-log` record. You can also paste +a JSON object into the input; arrays call `process-batch` with `list`. A single-record filter +can return no record, which stops subsequent steps. **Restore sample** switches back to the full batch. + The JSON input is editable. Every run starts from that input, so a previous run's filters, scores, and reports never leak into the next run. **Restore sample** resets the input; **Clear pipeline** removes the steps. Invalid uploads or options show an error on the page and can be corrected without a reload. @@ -71,8 +75,10 @@ It reads that Wasm back through the same loader as an upload and validates `desc the step. Code runs inside the shared QuickJS engine. The JavaScript engine is reused as you build and load new components. -The editor and compiler load from versioned CDN URLs, so browser authoring needs internet access. -Your source stays in the browser; no compilation requests are sent to a server. If the editor CDN is +The editor and compiler load from versioned [jsDelivr](https://www.jsdelivr.com/) CDN URLs, so browser +authoring needs internet access. An import map pins CodeMirror's shared dependencies to one version +each so its extension identity checks work. Your source stays in the browser; no compilation requests +are sent to a server. If the editor CDN is unavailable, the page keeps a plain source text area. Build errors appear beside the editor and leave the pipeline intact. Browser builds validate syntax and exports; full TypeScript checking for the checked-in components and editor template is provided by `pnpm run check`. @@ -118,18 +124,29 @@ export is checked against its generated interface, for example: ```ts import type { processor as ProcessorInterface } from 'example:log-pipeline/pipeline'; +import { getDataset } from '../shared/state.ts'; export const processor: typeof ProcessorInterface = { describe() { - /* name, description, defaultOptions */ + return { name: 'My processor', description: 'Process access logs.', defaultOptions: '{}' }; + }, + processBatch(logs, options) { + const data = getDataset(); + data.logs = logs; + /* parse options and update shared data */ + return { logs: data.logs, summary: `Processed ${logs.length} logs` }; }, - process(options) { - /* update shared data and return a summary */ + processSingle(log, options) { + const { logs, summary } = processor.processBatch([log], options); + return { log: logs[0], summary }; }, }; ``` -The generated `AccessLog` type also describes the shared records. These declarations are checked in +The shared WIT `types` interface defines `access-log`. The dataset and processor interfaces both use +it: `reset` accepts a list, `snapshot` returns typed logs, `process-batch` accepts and returns a list, +and `process-single` accepts one record and returns an optional record. The generated `AccessLog` +TypeScript type also describes the shared records. These declarations are checked in for editor support and regenerated by `build`; Jco bundles TypeScript directly, so there is no separate JavaScript compilation step. @@ -172,12 +189,20 @@ The shared [`createPipeline`](./demo/shared/pipeline.js) runner instantiates the platform adapter is a function that reads bytes from a URL: Node.js supplies `readFile`, and the browser supplies `fetch`. Both entry points use the same script reader and execution code. -For each run, the runner loads the dataset script and calls `dataset.reset(input)`. For each step, it -loads that step's source with `scriptLoader.load`, calls `processor.process(options)`, then reloads the -dataset script to read a snapshot. Loading scripts replaces the active exports, but retains the -engine's JavaScript realm. The working dataset lives on that realm's `globalThis`, shared through -[`components/shared/state.ts`](./components/shared/state.ts); it is not passed back and forth between -processors as JSON. Only input, options, summaries, and output snapshots cross the host boundary. +For each run, the host parses the JSON input into records. The runner loads the dataset script and +calls `dataset.reset(logs)`. `runBatch(logs, stages)` calls each processor's `processBatch(logs, options)` +and passes its returned records to the next step. `runSingle(log, stages)` instead calls +`processSingle(log, options)` and stops if a filter returns no record. The checked-in single-record +implementations reuse the batch algorithm with a one-element list. Batch statistics, such as repeated +IP counts, therefore apply only to the records in that call. + +Logs cross the WIT boundary as records, including optional country and risk-score annotations. +Options and extensible reports remain JSON strings; log records are never encoded as JSON inside the +engine. After each step, the runner reloads the dataset script to read a typed snapshot. Loading +scripts replaces the active exports, but retains the engine's JavaScript realm. The dataset and +reports live on that realm's `globalThis`, shared through +[`components/shared/state.ts`](./components/shared/state.ts). Reports survive switching processors +and reset at the beginning of each run. Both hosts format the final records as JSON for display. > [!NOTE] > Scripts in one engine share a realm and can access each other's data. Use trusted scripts together; @@ -202,15 +227,19 @@ Choose the Node.js stages and their execution order with positional arguments or pnpm run run top-pages geoip PIPELINE_STAGES=geoip,top-pages pnpm run run PIPELINE_STAGES= pnpm run run +pnpm run run --single risk-score top-pages ``` Arguments take precedence over the environment variable. With neither specified, the demo runs `geoip`, `risk-score`, then `top-pages`. An empty environment value runs an empty pipeline. Stage names are the filenames in `dist/components/` without `.wasm`; unknown names report the available stages and exit with a nonzero status. Each selected stage uses its own default options. +`--single` processes the first sample record through `process-single`; the default processes the +whole sample array through `process-batch`. -The entry point exports an async `main()` and calls it only when executed directly, so importing -`demo/node.mjs` does not run the demo. +The entry point exports an async `main({ stageNames, mode })` and calls it only when executed directly, +so importing `demo/node.mjs` does not run the demo. CLI and environment parsing happen in the +direct-execution block at the bottom; `main` receives named options. ### Add a processor @@ -227,8 +256,9 @@ pnpm run test:node pnpm run test:browser ``` -The tests exercise the built Wasm components, shared data across script loads, ordering and options, -repeat runs, empty input, invalid uploads, and recovery from errors. Node.js tests also cover CLI and +The tests exercise typed batches and single records through the built Wasm components, optional +filter results, shared data across script loads, ordering and options, repeat runs, empty input, +invalid uploads, and recovery from errors. Node.js tests also cover CLI and environment stage selection, precedence, unknown names, and the direct-execution guard. The Puppeteer test uploads actual `.wasm` files, edits the highlighted TypeScript editor, builds custom components, loads the downloadable bytes in Node.js, uploads them again, and checks recovery from syntax errors. diff --git a/examples/components/script-only-pipeline/components/dataset/dataset.ts b/examples/components/script-only-pipeline/components/dataset/dataset.ts index 0b3bf9b48..a9f34b2f8 100644 --- a/examples/components/script-only-pipeline/components/dataset/dataset.ts +++ b/examples/components/script-only-pipeline/components/dataset/dataset.ts @@ -1,28 +1,11 @@ import type { dataset as DatasetInterface } from 'example:log-pipeline/pipeline'; -import type { AccessLog } from 'example:log-pipeline/dataset'; import { getDataset, resetDataset } from '../shared/state.ts'; export const dataset: typeof DatasetInterface = { - reset(json) { - let rows: unknown; - try { - rows = JSON.parse(json); - } catch { - throw 'Input must be valid JSON'; - } - if (!Array.isArray(rows)) { - throw 'Input must be a JSON array of access logs'; - } - const logs: AccessLog[] = rows.map((row, index) => { - if ( - !row || - typeof row !== 'object' || - !['timestamp', 'ip', 'method', 'path'].every((key) => typeof row[key] === 'string') || - !Number.isInteger(row.status) || - row.status < 100 || - row.status > 599 - ) { - throw `Log ${index + 1} needs timestamp, ip, method, path, and an HTTP status (100–599)`; + reset(rows) { + const logs = rows.map((row, index) => { + if (row.status < 100 || row.status > 599) { + throw `Log ${index + 1} needs an HTTP status (100–599)`; } // Start each run from raw logs; annotations belong to the processors. return { @@ -38,6 +21,7 @@ export const dataset: typeof DatasetInterface = { resetDataset(logs); }, snapshot() { - return JSON.stringify(getDataset()); + const { logs, reports } = getDataset(); + return { logs, reportsJson: JSON.stringify(reports) }; }, }; diff --git a/examples/components/script-only-pipeline/components/geoip/geoip.ts b/examples/components/script-only-pipeline/components/geoip/geoip.ts index f18830331..84f0fc9cb 100644 --- a/examples/components/script-only-pipeline/components/geoip/geoip.ts +++ b/examples/components/script-only-pipeline/components/geoip/geoip.ts @@ -17,12 +17,13 @@ export const processor: typeof ProcessorInterface = { defaultOptions: JSON.stringify({ country: 'DE' }), }; }, - process(json) { + processBatch(logs, json) { const { country = 'DE' } = parseOptions(json); if (typeof country !== 'string' || !['all', 'US', 'DE', 'JP', 'unknown'].includes(country)) { throw 'country must be all, US, DE, JP, or unknown'; } const data = getDataset(); + data.logs = logs; const before = data.logs.length; data.logs = data.logs .map((log) => ({ @@ -30,6 +31,10 @@ export const processor: typeof ProcessorInterface = { country: countries.get(log.ip.split('.').slice(0, 3).join('.')) ?? 'unknown', })) .filter((log) => country === 'all' || log.country === country); - return `GeoIP: ${before} → ${data.logs.length} logs (${country})`; + return { logs: data.logs, summary: `GeoIP: ${before} → ${data.logs.length} logs (${country})` }; + }, + processSingle(log, options) { + const { logs, summary } = processor.processBatch([log], options); + return { log: logs[0], summary }; }, }; diff --git a/examples/components/script-only-pipeline/components/risk-score/risk-score.ts b/examples/components/script-only-pipeline/components/risk-score/risk-score.ts index c84abfa97..644f0e73c 100644 --- a/examples/components/script-only-pipeline/components/risk-score/risk-score.ts +++ b/examples/components/script-only-pipeline/components/risk-score/risk-score.ts @@ -9,12 +9,13 @@ export const processor: typeof ProcessorInterface = { defaultOptions: JSON.stringify({ minScore: 0 }), }; }, - process(json) { + processBatch(logs, json) { const { minScore = 0 } = parseOptions(json); if (typeof minScore !== 'number' || !Number.isInteger(minScore) || minScore < 0 || minScore > 100) { throw 'minScore must be an integer from 0 to 100'; } const data = getDataset(); + data.logs = logs; const requests = new Map(); for (const log of data.logs) { requests.set(log.ip, (requests.get(log.ip) ?? 0) + 1); @@ -30,6 +31,10 @@ export const processor: typeof ProcessorInterface = { ), })) .filter((log) => log.riskScore >= minScore); - return `Risk: scored logs and kept ${data.logs.length} with score ≥ ${minScore}`; + return { logs: data.logs, summary: `Risk: scored logs and kept ${data.logs.length} with score ≥ ${minScore}` }; + }, + processSingle(log, options) { + const { logs, summary } = processor.processBatch([log], options); + return { log: logs[0], summary }; }, }; diff --git a/examples/components/script-only-pipeline/components/shared/state.ts b/examples/components/script-only-pipeline/components/shared/state.ts index a3ad4fbb1..2f8fe4146 100644 --- a/examples/components/script-only-pipeline/components/shared/state.ts +++ b/examples/components/script-only-pipeline/components/shared/state.ts @@ -1,4 +1,4 @@ -import type { AccessLog } from 'example:log-pipeline/dataset'; +import type { AccessLog } from 'example:log-pipeline/types'; export interface Dataset { logs: AccessLog[]; diff --git a/examples/components/script-only-pipeline/components/top-pages/top-pages.ts b/examples/components/script-only-pipeline/components/top-pages/top-pages.ts index eed8e70d7..6b2a28d42 100644 --- a/examples/components/script-only-pipeline/components/top-pages/top-pages.ts +++ b/examples/components/script-only-pipeline/components/top-pages/top-pages.ts @@ -9,12 +9,13 @@ export const processor: typeof ProcessorInterface = { defaultOptions: JSON.stringify({ limit: 5 }), }; }, - process(json) { + processBatch(logs, json) { const { limit = 5 } = parseOptions(json); if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > 100) { throw 'limit must be an integer from 1 to 100'; } const data = getDataset(); + data.logs = logs; const counts = new Map(); for (const log of data.logs) { counts.set(log.path, (counts.get(log.path) ?? 0) + 1); @@ -24,6 +25,10 @@ export const processor: typeof ProcessorInterface = { .sort((a, b) => b.visits - a.visits || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) .slice(0, limit); data.reports.topPages = pages; - return `Pages: counted ${data.logs.length} logs across ${counts.size} paths`; + return { logs: data.logs, summary: `Pages: counted ${data.logs.length} logs across ${counts.size} paths` }; + }, + processSingle(log, options) { + const { logs, summary } = processor.processBatch([log], options); + return { log: logs[0], summary }; }, }; diff --git a/examples/components/script-only-pipeline/demo/browser.js b/examples/components/script-only-pipeline/demo/browser.js index 0ccfe46af..d99c2c90f 100644 --- a/examples/components/script-only-pipeline/demo/browser.js +++ b/examples/components/script-only-pipeline/demo/browser.js @@ -117,15 +117,16 @@ $('#components').addEventListener('change', async (event) => { $('#run').addEventListener('click', () => { invalidate(); try { - const result = pipeline.run( - $('#input').value, - stages.map((stage) => ({ - script: stage.script, - options: JSON.parse(stage.options), - })), - ); + const input = JSON.parse($('#input').value); + const configuredStages = stages.map((stage) => ({ + script: stage.script, + options: JSON.parse(stage.options), + })); + const result = Array.isArray(input) + ? pipeline.runBatch(input, configuredStages) + : pipeline.runSingle(input, configuredStages); $('#output').textContent = JSON.stringify({ logs: result.logs, reports: result.reports }, null, 2); - $('#count').textContent = `${result.logs.length} logs / ${stages.length} steps`; + $('#count').textContent = `${result.logs.length} logs / ${result.steps.length} steps`; $('#output-note').textContent = 'Every run starts with fresh input. Steps share data inside the engine.'; for (const step of result.steps) { const row = document.createElement('li'); @@ -145,6 +146,10 @@ $('#clear').addEventListener('click', () => { status('Pipeline cleared · upload a component to start again'); }); $('#restore').addEventListener('click', restoreSample); +$('#single-input').addEventListener('click', () => { + $('#input').value = JSON.stringify(sample[1], null, 2); + invalidate(); +}); $('#input').addEventListener('input', invalidate); function restoreSample() { $('#input').value = JSON.stringify(sample, null, 2); diff --git a/examples/components/script-only-pipeline/demo/custom-component.js b/examples/components/script-only-pipeline/demo/custom-component.js index 1a4597871..a4fcd029c 100644 --- a/examples/components/script-only-pipeline/demo/custom-component.js +++ b/examples/components/script-only-pipeline/demo/custom-component.js @@ -58,7 +58,7 @@ export async function createCustomComponent(name, source) { }); const exports = Object.values(result.metafile.outputs).flatMap((output) => output.exports); if (exports.length !== 1 || exports[0] !== 'processor') { - throw new Error('Export one processor object with describe() and process(options) methods'); + throw new Error('Export one processor object with describe(), processBatch(), and processSingle() methods'); } await $init; return scripts.createComponent({ diff --git a/examples/components/script-only-pipeline/demo/custom-template.ts b/examples/components/script-only-pipeline/demo/custom-template.ts index 4a6497d96..503a46346 100644 --- a/examples/components/script-only-pipeline/demo/custom-template.ts +++ b/examples/components/script-only-pipeline/demo/custom-template.ts @@ -9,13 +9,18 @@ export const processor: typeof ProcessorInterface = { defaultOptions: JSON.stringify({ minStatus: 400 }), }; }, - process(json) { + processBatch(logs, json) { const { minStatus = 400 } = parseOptions(json); if (typeof minStatus !== 'number' || minStatus < 100 || minStatus > 599) { throw 'minStatus must be a number from 100 to 599'; } const data = getDataset(); + data.logs = logs; data.logs = data.logs.filter((log) => log.status >= minStatus); - return `HTTP status: kept ${data.logs.length} logs with status >= ${minStatus}`; + return { logs: data.logs, summary: `HTTP status: kept ${data.logs.length} logs with status >= ${minStatus}` }; + }, + processSingle(log, options) { + const { logs, summary } = processor.processBatch([log], options); + return { log: logs[0], summary }; }, }; diff --git a/examples/components/script-only-pipeline/demo/editor.js b/examples/components/script-only-pipeline/demo/editor.js index a0eb0c8df..6be3a0a28 100644 --- a/examples/components/script-only-pipeline/demo/editor.js +++ b/examples/components/script-only-pipeline/demo/editor.js @@ -1,12 +1,9 @@ -const CODEMIRROR_URL = 'https://esm.sh/codemirror@6.0.2'; -const JAVASCRIPT_URL = 'https://esm.sh/@codemirror/lang-javascript@6.2.4'; - /** Enhance the source textarea without making the rest of the demo depend on a CDN. */ export async function mountEditor(textarea, parent) { - const [{ EditorView, basicSetup }, { javascript }] = await Promise.all([ - import(/* @vite-ignore */ CODEMIRROR_URL), - import(/* @vite-ignore */ JAVASCRIPT_URL), - ]); + const modules = ['codemirror', '@codemirror/lang-javascript']; + const [{ EditorView, basicSetup }, { javascript }] = await Promise.all( + modules.map((name) => import(/* @vite-ignore */ name)), + ); const view = new EditorView({ parent, doc: textarea.value, diff --git a/examples/components/script-only-pipeline/demo/index.html b/examples/components/script-only-pipeline/demo/index.html index 1a713f9c3..db394379f 100644 --- a/examples/components/script-only-pipeline/demo/index.html +++ b/examples/components/script-only-pipeline/demo/index.html @@ -6,6 +6,30 @@ Access log lab · Jco script-only components + + @@ -46,8 +70,9 @@

      Write your own component

      Write a TypeScript processor, create its .wasm component here, and add it to your pipeline.

      - Export a processor with describe() and process(options). Import getDataset and parseOptions from - pipeline:state to work with the shared logs and reports. No server build is needed. + Export a processor with describe(), processBatch(logs, options), and processSingle(log, options). + Import getDataset and parseOptions from pipeline:state to work with the shared logs and reports. No + server build is needed.

      @@ -69,8 +94,10 @@

      Write your own component

      01 / Input

      - + + +

      An array calls process-batch; one record calls process-single.

      Synthetic logs and invented locations for documentation IP ranges. Risk scores are illustrative.

      diff --git a/examples/components/script-only-pipeline/demo/node.mjs b/examples/components/script-only-pipeline/demo/node.mjs index 28f907910..56e041450 100644 --- a/examples/components/script-only-pipeline/demo/node.mjs +++ b/examples/components/script-only-pipeline/demo/node.mjs @@ -2,31 +2,26 @@ import { readFile, readdir } from 'node:fs/promises'; import { createPipeline } from './shared/pipeline.js'; import { readScript } from './shared/read-script.js'; -export async function main(args = process.argv.slice(2)) { - // CLI arguments take precedence over the comma-separated environment setting. - // An explicitly empty PIPELINE_STAGES runs the input through an empty pipeline. - const names = (args.length ? args : (process.env.PIPELINE_STAGES ?? 'geoip,risk-score,top-pages').split(',')) - .map((name) => name.trim()) - .filter(Boolean); +export async function main({ stageNames, mode = 'batch' }) { const components = new URL('../dist/components/', import.meta.url); const available = (await readdir(components)) .filter((file) => file.endsWith('.wasm') && !['engine.wasm', 'dataset.wasm'].includes(file)) .map((file) => file.slice(0, -'.wasm'.length)); - for (const name of names) { + for (const name of stageNames) { if (!available.includes(name)) { throw new Error(`Unknown stage "${name}". Available stages: ${available.join(', ')}`); } } const pipeline = await createPipeline(readFile); - const input = await readFile(new URL('./access-logs.json', import.meta.url), 'utf8'); + const input = JSON.parse(await readFile(new URL('./access-logs.json', import.meta.url), 'utf8')); const stages = []; - for (const name of names) { + for (const name of stageNames) { const script = await readScript(await readFile(new URL(`${name}.wasm`, components))); const description = pipeline.describe(script); stages.push({ script, options: JSON.parse(description.defaultOptions) }); } - const result = pipeline.run(input, stages); + const result = mode === 'single' ? pipeline.runSingle(input[0], stages) : pipeline.runBatch(input, stages); for (const step of result.steps) { console.log(step.summary); } @@ -34,7 +29,17 @@ export async function main(args = process.argv.slice(2)) { } if (import.meta.main) { - main().catch((error) => { + const args = process.argv.slice(2); + const mode = args.includes('--single') ? 'single' : 'batch'; + const stageArgs = args.filter((arg) => arg !== '--single'); + // CLI arguments take precedence over the comma-separated environment setting. + // An explicitly empty PIPELINE_STAGES runs the input through an empty pipeline. + const stageNames = ( + stageArgs.length ? stageArgs : (process.env.PIPELINE_STAGES ?? 'geoip,risk-score,top-pages').split(',') + ) + .map((name) => name.trim()) + .filter(Boolean); + main({ stageNames, mode }).catch((error) => { console.error(error.message ?? error); process.exitCode = 1; }); diff --git a/examples/components/script-only-pipeline/demo/shared/pipeline.js b/examples/components/script-only-pipeline/demo/shared/pipeline.js index 538be0bee..b146bbd63 100644 --- a/examples/components/script-only-pipeline/demo/shared/pipeline.js +++ b/examples/components/script-only-pipeline/demo/shared/pipeline.js @@ -11,9 +11,47 @@ export async function createPipeline(loadBytes) { const load = (script) => engine.scriptLoader.load(script.name, script.source); const snapshot = () => { loadDataset(); - return JSON.parse(engine.dataset.snapshot()); + const { logs, reportsJson } = engine.dataset.snapshot(); + return { + // Keep the demo's JSON output shape by omitting unset WIT options. + logs: logs.map(({ country, riskScore, ...log }) => ({ + ...log, + ...(country === undefined ? {} : { country }), + ...(riskScore === undefined ? {} : { riskScore }), + })), + reports: JSON.parse(reportsJson), + }; }; + function run(logs, stages, single = false) { + try { + loadDataset(); + engine.dataset.reset(logs); + const steps = []; + for (const { script, options } of stages) { + if (single && logs.length === 0) { + break; + } + load(script); + let summary; + if (single) { + const result = engine.processor.processSingle(logs[0], JSON.stringify(options)); + logs = result.log === undefined ? [] : [result.log]; + summary = result.summary; + } else { + const result = engine.processor.processBatch(logs, JSON.stringify(options)); + logs = result.logs; + summary = result.summary; + } + // The typed results cross the host boundary; reports stay in the engine. + steps.push({ name: script.name, summary, ...snapshot() }); + } + return { ...snapshot(), steps }; + } finally { + engine.scriptLoader.unload(); + } + } + return { describe(script) { if (script.engineWorld !== dataset.engineWorld() || !script.exports.includes('processor')) { @@ -26,22 +64,39 @@ export async function createPipeline(loadBytes) { engine.scriptLoader.unload(); } }, - run(input, stages) { - try { - loadDataset(); - engine.dataset.reset(input); - const steps = []; - for (const { script, options } of stages) { - load(script); - const summary = engine.processor.process(JSON.stringify(options)); - // Switching back to the dataset script does not lose the data. - // Only snapshots cross the host boundary; the working set stays in the engine. - steps.push({ name: script.name, summary, ...snapshot() }); - } - return { ...snapshot(), steps }; - } finally { - engine.scriptLoader.unload(); - } + runBatch(logs, stages) { + return run(validateLogs(logs), stages); + }, + runSingle(log, stages) { + return run(validateLogs([log]), stages, true); }, }; } + +/** Validate records before canonical ABI coercions can change malformed field values. */ +function validateLogs(rows) { + if (!Array.isArray(rows)) { + throw new Error('Input must be an array of access logs'); + } + return rows.map((row, index) => { + if ( + !row || + typeof row !== 'object' || + !['timestamp', 'ip', 'method', 'path'].every((key) => typeof row[key] === 'string') || + !Number.isInteger(row.status) || + row.status < 100 || + row.status > 599 + ) { + throw new Error(`Log ${index + 1} needs timestamp, ip, method, path, and an HTTP status (100–599)`); + } + return { + timestamp: row.timestamp, + ip: row.ip, + method: row.method, + path: row.path, + status: row.status, + country: undefined, + riskScore: undefined, + }; + }); +} diff --git a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts index 9cda9fb0a..17f6c4979 100644 --- a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts +++ b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-dataset.d.ts @@ -1,19 +1,19 @@ +/// declare module 'example:log-pipeline/dataset' { /** - * Replace the shared dataset with a JSON array of access-log records. + * Replace the shared dataset with raw logs, clearing annotations and reports. */ - export function reset(json: string): void; + export function reset(logs: Array): void; /** - * Read the current logs and reports as JSON without resetting them. + * Read the current logs and reports without resetting them. */ - export function snapshot(): string; - export interface AccessLog { - timestamp: string, - ip: string, - method: string, - path: string, - status: number, - country?: string, - riskScore?: number, + export function snapshot(): DatasetSnapshot; + export type AccessLog = import('example:log-pipeline/types').AccessLog; + export interface DatasetSnapshot { + logs: Array, + /** + * Processor-defined reports encoded as a JSON object. + */ + reportsJson: string, } } diff --git a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts index ec1f57202..16e961112 100644 --- a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts +++ b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-processor.d.ts @@ -1,12 +1,29 @@ +/// declare module 'example:log-pipeline/processor' { export function describe(): Description; /** - * Mutate the shared dataset and return a short summary, or an error. + * Process a host-provided batch, updating the shared dataset and reports. */ - export function process(options: string): string; + export function processBatch(logs: Array, options: string): BatchResult; + /** + * Process one host-provided log; filters may remove it. + */ + export function processSingle(log: AccessLog, options: string): SingleResult; + export type AccessLog = import('example:log-pipeline/types').AccessLog; export interface Description { name: string, description: string, defaultOptions: string, } + export interface BatchResult { + logs: Array, + summary: string, + } + export interface SingleResult { + /** + * None means the processor filtered out this log. + */ + log?: AccessLog, + summary: string, + } } diff --git a/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-types.d.ts b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-types.d.ts new file mode 100644 index 000000000..8d482f0a7 --- /dev/null +++ b/examples/components/script-only-pipeline/generated/types/interfaces/example-log-pipeline-types.d.ts @@ -0,0 +1,11 @@ +declare module 'example:log-pipeline/types' { + export interface AccessLog { + timestamp: string, + ip: string, + method: string, + path: string, + status: number, + country?: string, + riskScore?: number, + } +} diff --git a/examples/components/script-only-pipeline/generated/types/pipeline.d.ts b/examples/components/script-only-pipeline/generated/types/pipeline.d.ts index c98696d19..1b0a3830a 100644 --- a/examples/components/script-only-pipeline/generated/types/pipeline.d.ts +++ b/examples/components/script-only-pipeline/generated/types/pipeline.d.ts @@ -1,6 +1,8 @@ /// /// +/// declare module 'example:log-pipeline/pipeline' { + export * as types from 'example:log-pipeline/types'; // export example:log-pipeline/types export * as dataset from 'example:log-pipeline/dataset'; // export example:log-pipeline/dataset export * as processor from 'example:log-pipeline/processor'; // export example:log-pipeline/processor } diff --git a/examples/components/script-only-pipeline/test/browser.mjs b/examples/components/script-only-pipeline/test/browser.mjs index 8a075798e..affe0d29b 100644 --- a/examples/components/script-only-pipeline/test/browser.mjs +++ b/examples/components/script-only-pipeline/test/browser.mjs @@ -10,14 +10,14 @@ import { createPipeline } from '../demo/shared/pipeline.js'; import { readScript } from '../demo/shared/read-script.js'; const component = (name) => fileURLToPath(new URL(`../dist/components/${name}.wasm`, import.meta.url)); -const input = await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8'); +const input = JSON.parse(await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8')); const nodePipeline = await createPipeline(readFile); const nodeStages = []; for (const name of ['geoip', 'risk-score', 'top-pages']) { const script = await readScript(await readFile(component(name))); nodeStages.push({ script, options: JSON.parse(nodePipeline.describe(script).defaultOptions) }); } -const expected = nodePipeline.run(input, nodeStages); +const expected = nodePipeline.runBatch(input, nodeStages); const server = await createServer({ ...config, configFile: false, @@ -59,7 +59,7 @@ try { assert.equal((await page.$$('#pipeline > li')).length, 0); assert.equal(await page.$eval('#empty', (el) => el.hidden), false); await page.click('#run'); - assert.deepEqual(await output(page), { logs: JSON.parse(input), reports: {} }); + assert.deepEqual(await output(page), { logs: input, reports: {} }); // Download links serve actual component binaries, ready to upload again. for (const name of ['geoip', 'risk-score', 'top-pages']) { @@ -80,6 +80,16 @@ try { assert.deepEqual(await output(page), { logs: expected.logs, reports: expected.reports }); assert.equal((await page.$$('#steps > li')).length, 3); + await page.click('#single-input'); + await page.click('#run'); + const single = nodePipeline.runSingle(input[1], nodeStages); + assert.deepEqual(await output(page), { logs: single.logs, reports: single.reports }); + await setValue(page, '#input', JSON.stringify(input[0])); + await page.click('#run'); + assert.deepEqual(await output(page), { logs: [], reports: {} }); + assert.equal((await page.$$('#steps > li')).length, 1); + await page.click('#restore'); + await setValue(page, '#pipeline > li:nth-child(2) textarea', '{"minScore":70}'); await page.click('#run'); assert.equal((await output(page)).logs.length, 3); @@ -108,7 +118,7 @@ try { assert.equal(await page.$eval('#status', (el) => el.dataset.state), 'error'); await page.click('#restore'); await page.click('#run'); - assert.deepEqual(await output(page), { logs: JSON.parse(input), reports: {} }); + assert.deepEqual(await output(page), { logs: input, reports: {} }); assert.equal((await page.$$('#pipeline > li')).length, 0); await page.waitForFunction(() => document.querySelector('#source-editor').dataset.ready === 'true', { timeout: 60_000, @@ -143,10 +153,13 @@ try { ); const customScript = await readScript(bytes); const description = nodePipeline.describe(customScript); - const fromNode = nodePipeline.run(input, [ + const fromNode = nodePipeline.runBatch(input, [ { script: customScript, options: JSON.parse(description.defaultOptions) }, ]); assert.deepEqual(await output(page), { logs: fromNode.logs, reports: fromNode.reports }); + const customStage = { script: customScript, options: JSON.parse(description.defaultOptions) }; + assert.deepEqual(nodePipeline.runSingle(input[8], [customStage]).logs, [input[8]]); + assert.deepEqual(nodePipeline.runSingle(input[1], [customStage]).logs, []); const downloaded = join(downloads, 'http-status.wasm'); await writeFile(downloaded, bytes); await page.click('#clear'); @@ -209,6 +222,6 @@ async function editSource(page, source) { await page.keyboard.down('Control'); await page.keyboard.press('KeyA'); await page.keyboard.up('Control'); - await page.keyboard.insertText(source); + await page.keyboard.sendCharacter(source); await page.waitForFunction((source) => document.querySelector('#source').value === source, {}, source); } diff --git a/examples/components/script-only-pipeline/test/pipeline.mjs b/examples/components/script-only-pipeline/test/pipeline.mjs index 760eb0715..42c8a76b4 100644 --- a/examples/components/script-only-pipeline/test/pipeline.mjs +++ b/examples/components/script-only-pipeline/test/pipeline.mjs @@ -8,7 +8,7 @@ import { createPipeline } from '../demo/shared/pipeline.js'; import { readScript } from '../demo/shared/read-script.js'; const pipeline = await createPipeline(readFile); -const input = await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8'); +const input = JSON.parse(await readFile(new URL('../demo/access-logs.json', import.meta.url), 'utf8')); const scripts = {}; const stages = []; for (const name of ['geoip', 'risk-score', 'top-pages']) { @@ -22,11 +22,11 @@ function component(name) { } test('an empty pipeline returns the original logs', () => { - assert.deepEqual(pipeline.run(input, []), { logs: JSON.parse(input), reports: {}, steps: [] }); + assert.deepEqual(pipeline.runBatch(input, []), { logs: input, reports: {}, steps: [] }); }); test('three separately packaged scripts share annotations and reports in one engine', () => { - const result = pipeline.run(input, stages); + const result = pipeline.runBatch(input, stages); assert.equal(result.steps.length, 3); assert.equal(result.logs.length, 5); assert(result.logs.every((log) => log.country === 'DE')); @@ -41,11 +41,11 @@ test('three separately packaged scripts share annotations and reports in one eng ]); // The first step's snapshot predates scoring; snapshots are independent copies. assert(result.steps[0].logs.every((log) => log.riskScore === undefined)); - assert.deepEqual(pipeline.run(input, stages), result); + assert.deepEqual(pipeline.runBatch(input, stages), result); }); test('options and order change the results', () => { - const risky = pipeline.run(input, [ + const risky = pipeline.runBatch(input, [ stages[0], { script: scripts['risk-score'], options: { minScore: 70 } }, stages[2], @@ -55,7 +55,7 @@ test('options and order change the results', () => { { path: '/login', visits: 2 }, { path: '/admin', visits: 1 }, ]); - const countFirst = pipeline.run(input, [stages[2], stages[0]]); + const countFirst = pipeline.runBatch(input, [stages[2], stages[0]]); assert.equal(countFirst.logs.length, 5); assert.equal(countFirst.reports.topPages[0].visits, 4); assert.equal( @@ -64,15 +64,44 @@ test('options and order change the results', () => { ); }); +test('a host-provided single record crosses each processor and returns typed annotations', () => { + const log = { ...input[3], path: '/login/日本語' }; + const result = pipeline.runSingle(log, stages); + assert.equal(result.steps.length, 3); + assert.deepEqual(result.logs, [{ ...log, country: 'DE', riskScore: 50 }]); + assert.deepEqual(result.reports.topPages, [{ path: log.path, visits: 1 }]); + assert.deepEqual(result, pipeline.runBatch([log], stages)); + assert.deepEqual(pipeline.runSingle(log, []), { logs: [log], reports: {}, steps: [] }); + assert.equal(log.country, undefined, 'processing cannot mutate the host record'); +}); + +test('single-record filtering returns none and stops subsequent processors', () => { + const result = pipeline.runSingle(input[0], stages); + assert.deepEqual(result.logs, []); + assert.deepEqual(result.reports, {}); + assert.equal(result.steps.length, 1); + const risky = pipeline.runSingle(input[3], [ + stages[0], + { script: scripts['risk-score'], options: { minScore: 70 } }, + stages[2], + ]); + assert.deepEqual(risky.logs, []); + assert.equal(risky.steps.length, 2); + assert.equal(pipeline.runBatch(input, stages).logs.length, 5); +}); + test('empty datasets and a new run cannot inherit previous annotations or reports', () => { - pipeline.run(input, stages); - assert.deepEqual(pipeline.run('[]', stages).reports, { topPages: [] }); - assert.deepEqual(pipeline.run(input, []), { logs: JSON.parse(input), reports: {}, steps: [] }); + pipeline.runBatch(input, stages); + assert.deepEqual(pipeline.runBatch([], stages).reports, { topPages: [] }); + assert.deepEqual(pipeline.runBatch(input, []), { logs: input, reports: {}, steps: [] }); }); test('invalid input and options produce recoverable component errors', () => { - for (const invalid of ['{', '{}', '[{}]', '[{"status":999}]']) { - assert.throws(() => pipeline.run(invalid, stages)); + for (const invalid of [null, {}, [{}], [{ ...input[0], status: 999 }], [{ ...input[0], status: 65_736 }]]) { + assert.throws(() => pipeline.runBatch(invalid, stages)); + } + for (const invalid of [null, {}, { ...input[0], status: 200.5 }, { ...input[0], path: null }]) { + assert.throws(() => pipeline.runSingle(invalid, stages)); } for (const [name, options] of [ ['geoip', { country: 'XX' }], @@ -81,9 +110,10 @@ test('invalid input and options produce recoverable component errors', () => { ['top-pages', { limit: 0 }], ['top-pages', []], ]) { - assert.throws(() => pipeline.run(input, [{ script: scripts[name], options }])); + assert.throws(() => pipeline.runBatch(input, [{ script: scripts[name], options }])); + assert.throws(() => pipeline.runSingle(input[1], [{ script: scripts[name], options }])); } - assert.equal(pipeline.run(input, stages).logs.length, 5); + assert.equal(pipeline.runBatch(input, stages).logs.length, 5); }); test('uploads must be processor script-only components', async () => { @@ -96,9 +126,9 @@ test('uploads must be processor script-only components', async () => { test('different engine instances have separate datasets', async () => { const other = await createPipeline(readFile); - assert.equal(other.run('[]', stages).logs.length, 0); - assert.equal(pipeline.run(input, stages).logs.length, 5); - assert.equal(other.run(input, []).logs.length, 12); + assert.equal(other.runBatch([], stages).logs.length, 0); + assert.equal(pipeline.runBatch(input, stages).logs.length, 5); + assert.equal(other.runBatch(input, []).logs.length, 12); }); const exec = promisify(execFile); @@ -132,7 +162,7 @@ test('the Node CLI selects stages from arguments or ENV, with arguments taking p const empty = await runDemo([], { PIPELINE_STAGES: '' }); assert.equal(empty.summaries, ''); - assert.deepEqual(empty.result, { logs: JSON.parse(input), reports: {} }); + assert.deepEqual(empty.result, { logs: input, reports: {} }); }); test('the Node CLI reports unknown stages and exits unsuccessfully', async () => { @@ -143,6 +173,17 @@ test('the Node CLI reports unknown stages and exits unsuccessfully', async () => }); }); +test('the Node CLI can select single-record processing with stage arguments or ENV', async () => { + for (const [args, env] of [ + [['--single', 'risk-score', 'top-pages'], {}], + [['--single'], { PIPELINE_STAGES: 'risk-score,top-pages' }], + ]) { + const single = await runDemo(args, env); + assert.deepEqual(single.result.logs, [{ ...input[0], riskScore: 0 }]); + assert.deepEqual(single.result.reports.topPages, [{ path: '/', visits: 1 }]); + } +}); + test('importing the Node entry point does not run the demo', async () => { const { stdout, stderr } = await exec(process.execPath, [ '--input-type=module', diff --git a/examples/components/script-only-pipeline/wit/pipeline.wit b/examples/components/script-only-pipeline/wit/pipeline.wit index c31259ce9..77c32ca18 100644 --- a/examples/components/script-only-pipeline/wit/pipeline.wit +++ b/examples/components/script-only-pipeline/wit/pipeline.wit @@ -1,6 +1,6 @@ package example:log-pipeline; -interface dataset { +interface types { record access-log { timestamp: string, ip: string, @@ -10,26 +10,52 @@ interface dataset { country: option, risk-score: option, } +} + +interface dataset { + use types.{access-log}; + + record dataset-snapshot { + logs: list, + /// Processor-defined reports encoded as a JSON object. + reports-json: string, + } - /// Replace the shared dataset with a JSON array of access-log records. - reset: func(json: string) -> result<_, string>; - /// Read the current logs and reports as JSON without resetting them. - snapshot: func() -> string; + /// Replace the shared dataset with raw logs, clearing annotations and reports. + reset: func(logs: list) -> result<_, string>; + /// Read the current logs and reports without resetting them. + snapshot: func() -> dataset-snapshot; } interface processor { + use types.{access-log}; + record description { name: string, description: string, default-options: string, } + record batch-result { + logs: list, + summary: string, + } + + record single-result { + /// None means the processor filtered out this log. + log: option, + summary: string, + } + describe: func() -> description; - /// Mutate the shared dataset and return a short summary, or an error. - process: func(options: string) -> result; + /// Process a host-provided batch, updating the shared dataset and reports. + process-batch: func(logs: list, options: string) -> result; + /// Process one host-provided log; filters may remove it. + process-single: func(log: access-log, options: string) -> result; } world pipeline { + export types; export dataset; export processor; }