diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0efbda..d41752a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,11 +61,44 @@ jobs: run: node packages/codeupipe-ts/examples/pricing/main.ts - name: The linter on the TypeScript code of this repository run: npm run lint -w codeupipe-lint + - name: Install the Python package, for the judge + # conformance/run.py is a codeupipe pipeline (conformance/judging/). + run: python -m pip install -e packages/codeupipe - name: Conformance run: python conformance/run.py typescript:run typescript:static - name: The packages as npm makes them run: npm pack --workspaces --dry-run + rust: + name: Rust packages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + # The runner image has rustup and a stable toolchain. Nothing is + # downloaded from crates.io: the crates have no dependencies. + - name: Format + run: cargo fmt --all --check + - name: Clippy + run: cargo clippy --offline --all-targets -- -D warnings + - name: Tests + run: cargo test --offline + - name: Examples + run: | + cargo run --offline --example pricing -p codeupipe + cargo run --offline --example typed_pricing -p codeupipe + - name: Install the Python package, for the judge + # conformance/run.py is a codeupipe pipeline (conformance/judging/). + run: python -m pip install -e packages/codeupipe + - name: Conformance + run: python conformance/run.py rust:run + - name: The oldest Rust that the crate supports (rust-version) + run: | + rustup toolchain install 1.75 --profile minimal + cargo +1.75 test --offline -p codeupipe + export: name: Export against dynamic run runs-on: ubuntu-latest @@ -104,11 +137,12 @@ jobs: /tmp/installed/bin/python -m codeupipe_lint.invariants --rules /tmp/installed/bin/python -m codeupipe_lint.anonymous_steps --rules - speed: - name: Speed check (does not block) + benchmark: + name: Benchmark (numbers only, never fails the run) runs-on: ubuntu-latest - # The baseline was recorded on a different machine. This job shows how - # far a CI machine is from it; it does not fail the run until that is known. + # A shared CI machine is too noisy to judge speed, so nothing here is + # compared with a baseline. The job measures and keeps the numbers: in + # the job summary and as an artifact. continue-on-error: true steps: - uses: actions/checkout@v5 @@ -119,5 +153,13 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e packages/codeupipe -e packages/codeupipe-lint -r requirements-dev.txt - - name: Compare with the recorded baseline - run: python conformance/bench.py --check + - name: Measure + run: | + set -o pipefail + python conformance/bench.py | tee -a "$GITHUB_STEP_SUMMARY" + - name: Keep the numbers + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark + path: conformance/results/bench-latest.* diff --git a/.gitignore b/.gitignore index 9fe902d..70a8784 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ conformance/results/ .venv-*/ scorecard/results/ node_modules/ + +/target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c989904 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "codeupipe" +version = "0.1.0" + +[[package]] +name = "cup-conformance-rust" +version = "0.1.0" +dependencies = [ + "codeupipe", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8d7a3de --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +# The Cargo workspace of the Rust packages and the Rust conformance runner. +# It is not a package. +[workspace] +resolver = "2" +members = ["packages/codeupipe-rs", "conformance/runners/rust"] diff --git a/README.md b/README.md index 9dbdfca..c41ab18 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you know Unix, these are close to things you know. | Hook | A git hook: code that runs before and after a step | | State | `$?` and `PIPESTATUS` | -This repository has five packages. +This repository has six packages. | Package | Contents | Reference | |---|---|---| @@ -37,6 +37,7 @@ This repository has five packages. | `codeupipe` for TypeScript | The eight primitives. No dependencies | [packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) | | `codeupipe-lint` for TypeScript | A linter for pipeline code in TypeScript | [packages/codeupipe-lint-ts/README.md](packages/codeupipe-lint-ts/README.md) | | `codeupipe-compile` for TypeScript | Writes a pipeline as plain TypeScript with no import of codeupipe | [packages/codeupipe-compile-ts/README.md](packages/codeupipe-compile-ts/README.md) | +| `codeupipe` for Rust | The primitives, 0.1.0. No dependencies. No compiler or linter yet | [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) | ## Install @@ -86,6 +87,11 @@ compatible. They are not on npm yet: [packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) says how to install them from this repository. +The Rust crate is new too, version `0.1.0` on the same terms. It is not on +crates.io yet: +[packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) says how to +use it from this repository. + ## Example [examples/pricing/](examples/pricing/) has one class in each file. The @@ -141,7 +147,7 @@ class PricingPipeline(Pipeline): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) ``` @@ -187,7 +193,7 @@ An editable install needs pip 21.3 or later. | `python -m codeupipe_lint.anonymous_steps` | Runs the anonymous-step linter. Paths come from `cup.config.json` | | `python -m codeupipe_lint.invariants` | Runs the invariant linter. Paths come from `cup.config.json` | | `python -m codeupipe.compile examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing` | Compiles the example to `build/pricing/src/steps/` | -| `python conformance/bench.py --check` | Compares benchmark numbers with the recorded baseline. Fails outside 25% | +| `python conformance/bench.py` | Measures every target on the benchmark scenarios. It reports numbers and judges nothing | | `python -m scorecard.differential` | Compares the compiled code with the dynamic run on random pipelines | ## Layout @@ -199,6 +205,7 @@ An editable install needs pip 21.3 or later. | `packages/codeupipe-ts/` | The TypeScript runtime, with its tests and its example | | `packages/codeupipe-lint-ts/` | The TypeScript linter, with its tests | | `packages/codeupipe-compile-ts/` | The TypeScript compiler, with its tests | +| `packages/codeupipe-rs/` | The Rust runtime, with its tests and its example | | `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runners, the benchmark | | `tests/` | The tests for the two Python packages | | `examples/pricing/` | The example, with its test in `tests/examples/` | @@ -207,14 +214,17 @@ An editable install needs pip 21.3 or later. The root `pyproject.toml` is not a package. It holds the test settings. The root `package.json` is not a package: it makes the three TypeScript packages -one npm workspace. +one npm workspace. The root `Cargo.toml` is not a package either: it makes +the Rust runtime and its conformance runner one Cargo workspace. ## Other languages The contract and the conformance cases do not depend on a language. Python is the first runtime and TypeScript is the second ([packages/codeupipe-ts/](packages/codeupipe-ts/)), with its own linter and -its own compiler. A runtime in another language must implement +its own compiler. Rust is the third +([packages/codeupipe-rs/](packages/codeupipe-rs/)); its plan is +[docs/rust-plan.md](docs/rust-plan.md). A runtime in another language must implement the contract and add a conformance runner. ## License diff --git a/conformance/API.md b/conformance/API.md index ca65db6..a822452 100644 --- a/conformance/API.md +++ b/conformance/API.md @@ -14,10 +14,10 @@ parameters, their defaults, and how they fail. | State | records what happened | | Pipeline | runs the steps in order | -Python and TypeScript implement it. Each runtime has the same contract, +Python, TypeScript and Rust implement it. Each runtime has the same contract, with identifier casing in the style of its language, and the same conformance cases judge it. The tables below show the Python names. The -TypeScript names and differences are in "Runtimes". +TypeScript and Rust names and differences are in "Runtimes". ## Principles @@ -71,6 +71,24 @@ Rules: - A Tap, a Hook or a Predicate that wants its own changed version, or that keeps the Payload for later, takes a `copy()`. +### A type of your own + +The Payload is the default, and the form that this contract, the +conformance cases and the compilers use: a map that any developer can read, +which shows that anything can go there. A pipeline can carry a type of your +own instead, such as the class or struct that your steps would use anyway. +Its Filters, Predicates, Taps and Hooks then take that type. + +- **The Pipeline asks one thing of it: `copy()`**, for the copy that `run` + works on. Every rule above stays the same. +- **Lineage is kept only by a type that has `_stamp(label)`** (`stamp` in + Rust). For any other type, `observe(lineage=True)` does nothing: the run + goes on, and `state.executed` still lists the steps. +- **The compilers read Payload operations only.** A step that uses a type of + your own stays dynamic; the TypeScript compiler refuses it. + +How each runtime spells it is in "Runtimes". + ## Filter `call(payload) -> payload`. A failure raises. @@ -217,15 +235,16 @@ map and fails the comparison. ## Runtimes -| | Python | TypeScript | -|---|---|---| -| Package | `packages/codeupipe` | `packages/codeupipe-ts` | -| Names | snake_case | camelCase: `addFilter`, `addValve`, `toDict`, `withTrace`, `traceId`, `onError` | -| Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | -| Absent | `None` | `null` | -| Label of a step | `type(step).__name__` | `step.constructor.name` | -| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | -| Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | +| | Python | TypeScript | Rust | +|---|---|---|---| +| Package | `packages/codeupipe` | `packages/codeupipe-ts` | `packages/codeupipe-rs` (crate `codeupipe`) | +| Names | snake_case | camelCase: `addFilter`, `addValve`, `toDict`, `withTrace`, `traceId`, `onError` | snake_case, as Python | +| Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | `run` is sync, and `run_with_state` for the caller's State. No async run yet | +| Absent | `None` | `null` | `None` from `get`; `Value::Null` is a JSON null | +| Label of a step | `type(step).__name__` | `step.constructor.name` | The name of the step's type, from `std::any::type_name` | +| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | None yet | +| Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | None yet. The compiler refuses several of the shapes that the linters find | +| A type of your own | Any object with `copy()` | `class Order implements PipelineData`, then `new Pipeline()`, `Filter` | `impl PipelineData for Order {}`, then `Pipeline`, `Filter` | Notes on TypeScript: @@ -240,3 +259,23 @@ Notes on TypeScript: - **State and concurrent runs.** Two runs of one pipeline at the same time each write to their own State. `state` is the State of the run that started last. + +Notes on Rust: + +- **Refused when added, at compile time.** A closure is not a step, the + arguments of `add_valve` are in one order only, and a Tap, a Hook or a + Predicate gets `&Payload`, so it cannot write. Each is a type error. +- **A Filter changes the Payload in place.** `call(&self, &mut Payload)` + returns `Result<(), StepError>`. A failure is an `Err`; a panic is not a + failure of the step and is not caught. +- **Values.** A value is a `Value`: null, a bool, a number (`f64`), a string, + a list or a map that keeps the order of its keys, as JSON has. +- **A State for each run.** `run(&self, …)` takes the pipeline by shared + reference, so a step can hold the pipeline that it is in (an `Rc`) and run + it again. The run with the caller's State is + `run_with_state(&input, &mut state)`. `state()` gives an `Rc`: the + State of the run that started last, once it has ended, as a snapshot.md](../docs/rust-plan.md) has the + plan. +- **A type of your own** is a type parameter of each primitive, with + `Payload` as its default, so a step for one type in a pipeline of another + is a compile error. TypeScript does the same with its type check. diff --git a/conformance/BENCH.md b/conformance/BENCH.md index a9c241e..7d3dd2e 100644 --- a/conformance/BENCH.md +++ b/conformance/BENCH.md @@ -5,25 +5,28 @@ A snapshot of `python conformance/bench.py`. Scenarios: Every target's output is checked against the others before its time counts. Apple Silicon, one run. -These numbers are a snapshot for reading. The numbers that the speed check -compares are in `bench/baseline.json` (`python conformance/bench.py --check`, -[README.md](README.md)). +These numbers are a snapshot for reading. Nothing fails because of them: CI +measures again on every run and keeps the numbers, and does not compare them +([README.md](README.md)). `python:run` is the dynamic run. The three `static-*` targets are the compiles: `static-observe` records State and calls Hooks as `run` does; the two fast modes do neither, and refuse the `hook` scenario rather than silently dropping the Hook. +`rust:run` is the dynamic run of the Rust crate, built with `--release`. It +records State and calls Hooks as `python:run` does. + µs per run (best of 5 × 20,000, after 20,000 warmup runs); lower is better. -| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | -|---|---|---|---|---|---| -| `micro-pass` | 10 | 4.92 | 0.838 | 0.580 | 1.95 | -| `valve-gated` | 10 | 5.61 | 1.15 | 0.900 | 2.30 | -| `taps` | 13 | 5.21 | 0.887 | 0.629 | 2.07 | -| `hook` | 10 | 5.96 | — | — | 3.03 | -| `heavy-allocation` | 50 | 17.14 | 2.80 | 1.49 | 7.22 | +| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | rust:run | +|---|---|---|---|---|---|---| +| `micro-pass` | 10 | 4.94 | 0.824 | 0.606 | 1.92 | 0.246 | +| `valve-gated` | 10 | 5.81 | 1.11 | 0.801 | 2.25 | 0.301 | +| `taps` | 13 | 5.34 | 0.899 | 0.602 | 2.03 | 0.256 | +| `hook` | 10 | 5.88 | — | — | 2.97 | 0.247 | +| `heavy-allocation` | 50 | 16.05 | 2.84 | 1.49 | 7.01 | 4.24 | Not measured (the target refuses the scenario): -- python:static-steps hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast mode. Compile with observe=True (top-level Hooks only), or remove them. -- python:static-inline hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast mode. Compile with observe=True (top-level Hooks only), or remove them. +- python:static-steps hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast static export. Export with observe=True (top-level Hooks only), or remove them. +- python:static-inline hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast static export. Export with observe=True (top-level Hooks only), or remove them. diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md index 14481c7..59cb9f5 100644 --- a/conformance/MATRIX.md +++ b/conformance/MATRIX.md @@ -5,8 +5,8 @@ live copy is `results/latest.md` (not committed). The command exits non-zero if any target has a ❌. **Every target passes every case, and nothing is unsupported.** The targets -are the Python dynamic run, async run and three compiled forms, and the -TypeScript run and its compiled form. +are the Python dynamic run, async run and three compiled forms, the +TypeScript run and its compiled form, and the Rust run. The fast modes (`python:static-steps`, `python:static-inline`, `typescript:static`) expose no State or Hooks, so they pass those cases as @@ -16,38 +16,38 @@ the case that asks for `state` on the Pipeline object. A compiled form is a function and has no such object. See [docs/compile.md](../docs/compile.md). -| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | -|---|---|---|---|---|---|---|---| -| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `state/the-caller-gives-the-state-of-a-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `state/a-run-that-starts-inside-another-run-has-its-own-state` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last` | ✅ | ✅ | ✅* | ✅* | ✅* | ✅ | ✅* | -| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | -| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | rust:run | +|---|---|---|---|---|---|---|---|---| +| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/the-caller-gives-the-state-of-a-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/a-run-that-starts-inside-another-run-has-its-own-state` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last` | ✅ | ✅ | ✅* | ✅* | ✅* | ✅ | ✅* | ✅ | +| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | - **python:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported - **python:run_async**: 30 pass, 0 pass*, 0 fail, 0 unsupported @@ -56,5 +56,6 @@ function and has no such object. See - **python:static-observe**: 29 pass, 1 pass*, 0 fail, 0 unsupported - **typescript:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported - **typescript:static**: 11 pass, 19 pass*, 0 fail, 0 unsupported +- **rust:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported ✅* passed everything the target can observe; it does not expose State or Hooks (the fast mode). ➖ the target cannot express the case. diff --git a/conformance/README.md b/conformance/README.md index 307cd0f..6ce43fe 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -27,14 +27,17 @@ The committed snapshot is [MATRIX.md](MATRIX.md). ([BENCH.md](BENCH.md)). The runner has a `bench` mode that times the scenarios in `bench/scenarios.json`. -**The speed check** compares each benchmark number with its recorded -baseline (`bench/baseline.json`): +**Speed is measured, not judged.** CI runs `bench.py` and keeps the numbers +(the job summary and an artifact), and never fails because of them: a shared +machine is too noisy for a speed gate. A comparison with a recorded baseline +(`bench/baseline.json`) is kept for a machine whose resources are fixed, such +as a constrained container. No test and no CI job runs it: ```bash -python conformance/bench.py --check # fails outside ±25% +python conformance/bench.py --check # outside ±25% of the baseline python conformance/bench.py --check --tolerance 15 -python conformance/bench.py --record # after a change in speed that you accept -python -m pytest -m perf # the same check, as a test +python conformance/bench.py --record # record the baseline +python conformance/bench.py rust:run --record # only the Rust numbers ``` A number is kept as a ratio to a reference workload (plain dict writes @@ -42,7 +45,9 @@ behind function calls) that the runner times beside it, so the check does not depend on how fast the machine is. Slower than the band is a regression. Faster than the band also fails, with the instruction to record again: otherwise the next regression would hide inside the gain. There is one -baseline for each Python version. Run it on a quiet machine: under heavy +baseline for each Python version, and one for the Rust run, which does not +depend on Python (the Rust runner times a reference of map writes of its own). +`--record` with targets named replaces only those targets. Run it on a quiet machine: under heavy load a number can leave the band. | Target | How it runs (all offline) | @@ -52,6 +57,7 @@ load a number can leave the band. | `python:static-observe` | `codeupipe.compile` with `observe=True`: State and Hooks judged too | | `typescript:run` | `packages/codeupipe-ts` Pipeline. Node runs the `.ts` runner as it is (Node 22.18 or later). With no arguments, `run.py` leaves this target out when the machine has no such Node | | `typescript:static` | `packages/codeupipe-compile-ts`: the pipeline as plain code, written to a file and imported (no State or Hooks: ✅\*). Needs `npm ci` in that package and a built core package; `run.py` leaves it out when they are not there | +| `rust:run` | `packages/codeupipe-rs` Pipeline. `cargo run --release --offline` builds the runner, which has no dependency to download. With no arguments, `run.py` leaves this target out when the machine has no `cargo` | ## Statuses @@ -137,7 +143,8 @@ never halts the pipeline). Where it is silent, they encode Python when the Valve skips its filter. 3. **`equals` across types.** Python's `==` makes `1 == true`; JSON does not. TypeScript uses `===`, which does not compare lists or maps by content. - No case depends on it yet. + Rust compares `Value` by content, and a number is never equal to a + boolean. No case depends on it yet. 4. **`truthy` across languages.** An empty list or map is false in Python - and true in JavaScript. Each runner uses the rule of its language. No - case depends on it yet. + and true in JavaScript. Each runner uses the rule of its language; Rust + has no rule of its own and follows Python. No case depends on it yet. diff --git a/conformance/bench.py b/conformance/bench.py index 575ef6d..6449b15 100644 --- a/conformance/bench.py +++ b/conformance/bench.py @@ -11,7 +11,11 @@ output it produced; this script refuses a result whose output differs from the others, so a fast wrong answer cannot win. -The speed check (--check). A recorded time is true for one machine, so each +The comparison (--check) is for a machine whose resources are fixed, such as +a constrained container. CI does not run it: a shared machine is too noisy +to judge speed, so CI only measures and keeps the numbers. + +A recorded time is true for one machine, so each number is kept as a ratio to a reference workload: plain Python dict writes and function calls, the work a pipeline does. The runner times the reference and the scenario back to back, so a slower or a busier machine slows both @@ -21,7 +25,8 @@ the baseline is out of date and must be recorded again, or the next regression would hide inside the gain. A baseline is kept for each Python version (bench/baseline.json), because interpreters differ in what they make -fast. +fast. The Rust run does not depend on Python, so it has one baseline, under +"rust". --record replaces only the targets that it measured. """ import json @@ -41,6 +46,7 @@ "python:static-steps": ("python:static-steps", "bench:static-steps"), "python:static-inline": ("python:static-inline", "bench:static-inline"), "python:static-observe": ("python:static-observe", "bench:static-observe"), + "rust:run": ("rust:run", "bench:run"), } @@ -62,6 +68,33 @@ def load_baseline(): return json.loads(BASELINE.read_text()) if BASELINE.exists() else {} +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def baseline_key(target, python): + """Where the baseline of ``target`` is kept. A Python target is kept per + Python version; the Rust run does not depend on Python and has its own.""" + return "rust" if target.startswith("rust:") else python + + +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def baseline_for(baseline, targets, python): + """The recorded ratios of ``targets``, each from its own key.""" + relative = {} + for target in targets: + row = baseline.get(baseline_key(target, python), {}).get("relative", {}).get(target) + if row is not None: + relative[target] = row + return {"relative": relative} + + +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def record_into(baseline, current, python): + """Put the measured targets into ``baseline``. The others are kept.""" + for kind in ("relative", "us_per_run"): + for target, row in current[kind].items(): + recorded = baseline.setdefault(baseline_key(target, python), {"relative": {}, "us_per_run": {}}) + recorded.setdefault(kind, {})[target] = row + + # cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def compare(baseline, current, tolerance=DEFAULT_TOLERANCE): """Every number that is not within ``tolerance`` percent of its baseline. @@ -116,7 +149,8 @@ def main(argv): argv = [a for a in argv if a not in ("--record", "--check")] scenarios = json.loads((HERE / "bench" / "scenarios.json").read_text()) - names = argv or list(BENCH_TARGETS) + runnable = conformance.default_targets() + names = argv or [name for name in BENCH_TARGETS if BENCH_TARGETS[name][0] in runnable] table, relative, reference, problems, refused = {}, {}, {}, [], [] for name in names: results, error = conformance.run_target(name, spec_for(name), scenarios) @@ -155,17 +189,17 @@ def main(argv): if record: baseline = load_baseline() - baseline[python_key()] = current + record_into(baseline, current, python_key()) BASELINE.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n") - print("Recorded the baseline for Python %s in %s." % (python_key(), BASELINE.relative_to(HERE.parent))) + print("Recorded the baseline of %s in %s." % (", ".join(names), BASELINE.relative_to(HERE.parent))) return 0 if check: - baseline = load_baseline().get(python_key()) - if baseline is None: + baseline = load_baseline() + if python_key() not in baseline and any(name.startswith("python:") for name in names): print("Speed check: no baseline for Python %s. Record one: python conformance/bench.py --record" % python_key()) return 1 - found = compare(baseline, current, tolerance) + found = compare(baseline_for(baseline, names, python_key()), current, tolerance) print(_report(found, tolerance)) return 1 if found else 0 return 0 diff --git a/conformance/bench/baseline.json b/conformance/bench/baseline.json index c903c9f..a4fb61c 100644 --- a/conformance/bench/baseline.json +++ b/conformance/bench/baseline.json @@ -114,5 +114,25 @@ "valve-gated": 1.7382603999999968 } } + }, + "rust": { + "relative": { + "rust:run": { + "heavy-allocation": 5.329424607074787, + "hook": 0.36049069077903273, + "micro-pass": 0.3347908632428459, + "taps": 0.37290526661608375, + "valve-gated": 0.40713696452472475 + } + }, + "us_per_run": { + "rust:run": { + "heavy-allocation": 4.31750625, + "hook": 0.2910354, + "micro-pass": 0.2790875, + "taps": 0.29723954999999996, + "valve-gated": 0.33209375 + } + } } } diff --git a/conformance/judging/cargo_probe.py b/conformance/judging/cargo_probe.py new file mode 100644 index 0000000..ecb029c --- /dev/null +++ b/conformance/judging/cargo_probe.py @@ -0,0 +1,10 @@ +"""CargoProbe: what this machine has for the Rust targets.""" + +import shutil + + +class CargoProbe: + """Says if there is a ``cargo`` to build and run the Rust runner.""" + + def present(self) -> bool: + return shutil.which("cargo") is not None diff --git a/conformance/judging/default_targets_filter.py b/conformance/judging/default_targets_filter.py index 7c3a182..44588fb 100644 --- a/conformance/judging/default_targets_filter.py +++ b/conformance/judging/default_targets_filter.py @@ -9,14 +9,16 @@ class DefaultTargetsFilter: Input keys: ``targets`` ({name: how to start its runner}), ``needs`` ({name: paths that must exist for the target}). Output keys: ``names``. A Python target always runs. A TypeScript target - needs a Node that runs ``.ts`` files, and the paths of ``needs``. + needs a Node that runs ``.ts`` files, and the paths of ``needs``. A Rust + target needs ``cargo``. - ``node_runs_typescript`` is a function with no arguments that says if - the machine has such a Node. + ``node_runs_typescript`` and ``has_cargo`` are functions with no + arguments that say if the machine has such a Node, and a ``cargo``. """ - def __init__(self, node_runs_typescript): + def __init__(self, node_runs_typescript, has_cargo=lambda: False): self.node_runs_typescript = node_runs_typescript + self.has_cargo = has_cargo def call(self, payload: Payload) -> Payload: return payload.insert("names", self.choose(payload.get("targets"), payload.get("needs", {}))) @@ -25,6 +27,8 @@ def choose(self, targets, needs) -> list: return [name for name in targets if self.can_run(name, needs)] def can_run(self, name, needs) -> bool: + if name.startswith("rust:"): + return self.has_cargo() if not name.startswith("typescript:"): return True return self.node_runs_typescript() and all(path.exists() for path in needs.get(name, [])) diff --git a/conformance/judging/judge_pipeline.py b/conformance/judging/judge_pipeline.py index 72f638d..87afcb0 100644 --- a/conformance/judging/judge_pipeline.py +++ b/conformance/judging/judge_pipeline.py @@ -22,13 +22,14 @@ class JudgePipeline(Pipeline): Output keys: ``report``, ``matrix``, ``markdown``, ``exit_code``. ``launcher`` starts the runner of a target. ``node_runs_typescript`` says - if the machine can run the TypeScript targets. + if the machine can run the TypeScript targets, and ``has_cargo`` if it + can run the Rust targets. """ - def __init__(self, launcher, node_runs_typescript): + def __init__(self, launcher, node_runs_typescript, has_cargo=lambda: False): super().__init__() self.add_filter(LoadCasesFilter()) - self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript)) + self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript, has_cargo)) self.add_filter(RunTargetsFilter(launcher)) self.add_filter(JudgeResultsFilter()) self.add_filter(SummarizeFilter()) diff --git a/conformance/run.py b/conformance/run.py index 9e11f24..126b86f 100644 --- a/conformance/run.py +++ b/conformance/run.py @@ -3,9 +3,11 @@ python conformance/run.py # all targets python conformance/run.py python:run python:static-inline python conformance/run.py typescript:run + python conformance/run.py rust:run With no arguments, the TypeScript target runs only when the machine has -Node 22.18 or later. A target that is named and cannot run is a failure. +Node 22.18 or later, and the Rust target only when it has cargo. A target +that is named and cannot run is a failure. A runner executes cases and reports what happened. Only this file decides pass or fail, so every target is held to exactly the same comparison. @@ -30,6 +32,7 @@ from codeupipe import Payload # noqa: E402 +from judging.cargo_probe import CargoProbe # noqa: E402 from judging.default_targets_filter import DefaultTargetsFilter # noqa: E402 from judging.judge_pipeline import JudgePipeline # noqa: E402 from judging.judge_results_filter import JudgeResultsFilter # noqa: E402 @@ -57,6 +60,16 @@ def _typescript(target): } +def _rust(target): + # Built offline from the repository: the runner has no dependency to download. + return { + "cmd": ["cargo", "run", "--quiet", "--release", "--offline", + "--manifest-path", str(HERE / "runners/rust/Cargo.toml"), "--", target], + "env": {}, + "cwd": REPO, + } + + TARGETS = { "python:run": _python("run"), "python:run_async": _python("run_async"), @@ -65,6 +78,7 @@ def _typescript(target): "python:static-observe": _python("static-observe"), "typescript:run": _typescript("run"), "typescript:static": _typescript("static"), + "rust:run": _rust("run"), } # The static target runs the compile package. Its dependencies must be @@ -84,10 +98,16 @@ def _node_runs_typescript(): return NodeProbe().runs_typescript() +def _has_cargo(): + """Whether there is a cargo to build and run the Rust runner.""" + return CargoProbe().present() + + def default_targets(): - """Every target that this machine can run. TypeScript needs Node, and the - static TypeScript target needs its package to be installed.""" - return DefaultTargetsFilter(lambda: _node_runs_typescript()).choose(TARGETS, _TYPESCRIPT_NEEDS) + """Every target that this machine can run. TypeScript needs Node, the + static TypeScript target needs its package to be installed, and Rust + needs cargo.""" + return DefaultTargetsFilter(lambda: _node_runs_typescript(), lambda: _has_cargo()).choose(TARGETS, _TYPESCRIPT_NEEDS) def load_cases(): @@ -109,7 +129,7 @@ def judge(case, result): def main(argv): - pipeline = JudgePipeline(ProcessLauncher(), lambda: _node_runs_typescript()) + pipeline = JudgePipeline(ProcessLauncher(), lambda: _node_runs_typescript(), lambda: _has_cargo()) out = pipeline.run(Payload({"cases_dir": CASES, "results_dir": RESULTS, "targets": TARGETS, "needs": _TYPESCRIPT_NEEDS, "names": list(argv)})) print(out.get("markdown")) diff --git a/conformance/runners/rust/Cargo.toml b/conformance/runners/rust/Cargo.toml new file mode 100644 index 0000000..f74955c --- /dev/null +++ b/conformance/runners/rust/Cargo.toml @@ -0,0 +1,16 @@ +# The Rust conformance runner: runner < cases.json > results.json. +# It has no dependency to download, so run.py builds it with --offline. +[package] +name = "cup-conformance-rust" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +license = "Apache-2.0" +publish = false + +[[bin]] +name = "runner" +path = "src/main.rs" + +[dependencies] +codeupipe = { path = "../../../packages/codeupipe-rs" } diff --git a/conformance/runners/rust/src/bench.rs b/conformance/runners/rust/src/bench.rs new file mode 100644 index 0000000..ed0ec8e --- /dev/null +++ b/conformance/runners/rust/src/bench.rs @@ -0,0 +1,85 @@ +//! The bench mode: times the scenarios as the Python runner does, so that +//! conformance/bench.py can put the numbers side by side. +use std::collections::HashMap; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use codeupipe::{Map, Pipeline, Value}; + +const WARMUP: usize = 20_000; +const RUNS: usize = 20_000; +const REPEATS: usize = 5; + +const REFERENCE_KEYS: [&str; 10] = ["k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8", "k9"]; + +#[inline(never)] +fn reference_write(data: &mut HashMap, key: &str) { + *data.entry(key.to_string()).or_insert(0.0) += 1.0; +} + +/// The reference workload: ten map writes behind ten function calls, with no +/// framework. A pipeline's time is reported relative to it. +fn reference() -> HashMap { + let mut data = HashMap::from([("a".to_string(), 1.0)]); + for key in REFERENCE_KEYS { + reference_write(&mut data, black_box(key)); + } + data +} + +fn timed(f: &mut impl FnMut()) -> Duration { + let started = Instant::now(); + for _ in 0..RUNS { + f(); + } + started.elapsed() +} + +/// (µs per run, time relative to the reference workload). µs per run is the +/// best of REPEATS x RUNS. Each repeat times the reference and then `f`, +/// back to back, and the median of the ratios is reported. +fn measure(mut f: impl FnMut()) -> (f64, f64) { + let mut reference_run = || { + black_box(reference()); + }; + for _ in 0..WARMUP { + f(); + reference_run(); + } + let mut best = Duration::MAX; + let mut ratios = Vec::with_capacity(REPEATS); + for _ in 0..REPEATS { + let reference = timed(&mut reference_run); + let elapsed = timed(&mut f); + ratios.push(elapsed.as_secs_f64() / reference.as_secs_f64()); + best = best.min(elapsed); + } + ratios.sort_by(f64::total_cmp); + let us_per_run = best.as_secs_f64() / RUNS as f64 * 1e6; + (us_per_run, ratios[REPEATS / 2]) +} + +/// Time one scenario. `pipeline` is built from its steps; `input` is its input. +pub fn bench(id: Value, pipeline: Pipeline, input: codeupipe::Payload) -> Value { + let output = match pipeline.run(&input) { + Ok(output) => output.to_dict(), + Err(error) => { + return Value::Map(Map::from([ + ("id", id), + ( + "unsupported", + Value::from(format!("the scenario fails: {error}")), + ), + ])) + } + }; + let (us_per_run, relative) = measure(|| { + black_box(pipeline.run(black_box(&input)).ok()); + }); + Value::Map(Map::from([ + ("id", id), + ("us_per_run", Value::from(us_per_run)), + ("relative", Value::from(relative)), + ("output", Value::Map(output)), + ])) +} diff --git a/conformance/runners/rust/src/cup_ops/add_filter.rs b/conformance/runners/rust/src/cup_ops/add_filter.rs new file mode 100644 index 0000000..15abc46 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/add_filter.rs @@ -0,0 +1,19 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = get(key, 0) + by +pub struct AddFilter { + pub key: String, + pub by: f64, +} + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let n = match payload.get(&self.key) { + None => 0.0, + Some(Value::Number(n)) => *n, + Some(other) => return Err(format!("'{}' is not a number: {other:?}", self.key).into()), + }; + payload.insert(self.key.as_str(), n + self.by); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/append_filter.rs b/conformance/runners/rust/src/cup_ops/append_filter.rs new file mode 100644 index 0000000..9898c93 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/append_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = a copy of get(key), or of an empty list, with value added +pub struct AppendFilter { + pub key: String, + pub value: Value, +} + +impl Filter for AppendFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let mut list = match payload.get(&self.key) { + None | Some(Value::Null) => Vec::new(), + Some(Value::List(items)) => items.clone(), + Some(other) => return Err(format!("'{}' is not a list: {other:?}", self.key).into()), + }; + list.push(self.value.clone()); + payload.insert(self.key.as_str(), list); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/copy_filter.rs b/conformance/runners/rust/src/cup_ops/copy_filter.rs new file mode 100644 index 0000000..468fb8e --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/copy_filter.rs @@ -0,0 +1,15 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert to = get(from), null if missing +pub struct CopyFilter { + pub from: String, + pub to: String, +} + +impl Filter for CopyFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let value = payload.get(&self.from).cloned().unwrap_or(Value::Null); + payload.insert(self.to.as_str(), value); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/equals_predicate.rs b/conformance/runners/rust/src/cup_ops/equals_predicate.rs new file mode 100644 index 0000000..0696963 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/equals_predicate.rs @@ -0,0 +1,13 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// get(key) == value, by content; a missing key is null +pub struct EqualsPredicate { + pub key: String, + pub value: Value, +} + +impl Predicate for EqualsPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get(&self.key).unwrap_or(&Value::Null) == &self.value + } +} diff --git a/conformance/runners/rust/src/cup_ops/fail_filter.rs b/conformance/runners/rust/src/cup_ops/fail_filter.rs new file mode 100644 index 0000000..ec2ab08 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/fail_filter.rs @@ -0,0 +1,12 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// return an error that carries message +pub struct FailFilter { + pub message: String, +} + +impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err(self.message.clone().into()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/fail_tap.rs b/conformance/runners/rust/src/cup_ops/fail_tap.rs new file mode 100644 index 0000000..9f9d919 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/fail_tap.rs @@ -0,0 +1,12 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// return an error +pub struct FailTap { + pub message: String, +} + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err(self.message.clone().into()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/has_filter.rs b/conformance/runners/rust/src/cup_ops/has_filter.rs new file mode 100644 index 0000000..51d547f --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/has_filter.rs @@ -0,0 +1,15 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// insert to = has(key) +pub struct HasFilter { + pub key: String, + pub to: String, +} + +impl Filter for HasFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let has = payload.has(&self.key); + payload.insert(self.to.as_str(), has); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/merge_filter.rs b/conformance/runners/rust/src/cup_ops/merge_filter.rs new file mode 100644 index 0000000..16ae75c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/merge_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, Map, Payload, StepError}; + +/// merge(Payload(value)) +pub struct MergeFilter { + pub value: Map, +} + +impl Filter for MergeFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.merge(&Payload::from_map(self.value.clone())); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/mod.rs b/conformance/runners/rust/src/cup_ops/mod.rs new file mode 100644 index 0000000..cf2ac18 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/mod.rs @@ -0,0 +1,40 @@ +//! The steps of the cases: one type for each step, with the names that the +//! TypeScript runner uses, so that the labels are the same. +mod add_filter; +mod append_filter; +mod copy_filter; +mod equals_predicate; +mod fail_filter; +mod fail_tap; +mod has_filter; +mod merge_filter; +mod mutate_filter; +mod nested_pipeline; +mod noop_hook; +mod noop_tap; +mod record_hook; +mod record_tap; +mod remove_filter; +mod run_again_filter; +mod set_filter; +pub mod sink; +mod truthy_predicate; + +pub use add_filter::AddFilter; +pub use append_filter::AppendFilter; +pub use copy_filter::CopyFilter; +pub use equals_predicate::EqualsPredicate; +pub use fail_filter::FailFilter; +pub use fail_tap::FailTap; +pub use has_filter::HasFilter; +pub use merge_filter::MergeFilter; +pub use mutate_filter::MutateFilter; +pub use nested_pipeline::NestedPipeline; +pub use noop_hook::NoopHook; +pub use noop_tap::NoopTap; +pub use record_hook::RecordHook; +pub use record_tap::RecordTap; +pub use remove_filter::RemoveFilter; +pub use run_again_filter::RunAgainFilter; +pub use set_filter::SetFilter; +pub use truthy_predicate::TruthyPredicate; diff --git a/conformance/runners/rust/src/cup_ops/mutate_filter.rs b/conformance/runners/rust/src/cup_ops/mutate_filter.rs new file mode 100644 index 0000000..ffd1686 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/mutate_filter.rs @@ -0,0 +1,16 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert(key, value) as a statement of its own. A Rust Filter always +/// changes the Payload in place, so this is the same as SetFilter; the case +/// is kept so that every runtime runs it. +pub struct MutateFilter { + pub key: String, + pub value: Value, +} + +impl Filter for MutateFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert(self.key.as_str(), self.value.clone()); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/nested_pipeline.rs b/conformance/runners/rust/src/cup_ops/nested_pipeline.rs new file mode 100644 index 0000000..94da431 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/nested_pipeline.rs @@ -0,0 +1,9 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +/// A named pipeline type, for a pipeline that is a step of another. The +/// runner adds the steps of the case after it is made. +pub struct NestedPipeline; + +impl NamedPipeline for NestedPipeline { + fn add_steps(&self, _pipeline: &mut Pipeline) {} +} diff --git a/conformance/runners/rust/src/cup_ops/noop_hook.rs b/conformance/runners/rust/src/cup_ops/noop_hook.rs new file mode 100644 index 0000000..8b89641 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/noop_hook.rs @@ -0,0 +1,6 @@ +use codeupipe::Hook; + +/// A Hook that does nothing: the cost of having one. +pub struct NoopHook; + +impl Hook for NoopHook {} diff --git a/conformance/runners/rust/src/cup_ops/noop_tap.rs b/conformance/runners/rust/src/cup_ops/noop_tap.rs new file mode 100644 index 0000000..ef79509 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/noop_tap.rs @@ -0,0 +1,10 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// do nothing +pub struct NoopTap; + +impl Tap for NoopTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/record_hook.rs b/conformance/runners/rust/src/cup_ops/record_hook.rs new file mode 100644 index 0000000..f470f3c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/record_hook.rs @@ -0,0 +1,32 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Hook, Payload, StepError}; + +/// Records before:, after: and error:. Events of the +/// pipeline itself have an empty step. +pub struct RecordHook { + pub events: Rc>>, +} + +impl RecordHook { + fn record(&self, event: &str, step: Option<&str>) { + self.events + .borrow_mut() + .push(format!("{event}:{}", step.unwrap_or(""))); + } +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + self.record("before", step); + } + + fn after(&self, step: Option<&str>, _payload: &Payload) { + self.record("after", step); + } + + fn on_error(&self, step: Option<&str>, _error: &StepError, _payload: &Payload) { + self.record("error", step); + } +} diff --git a/conformance/runners/rust/src/cup_ops/record_tap.rs b/conformance/runners/rust/src/cup_ops/record_tap.rs new file mode 100644 index 0000000..692be4c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/record_tap.rs @@ -0,0 +1,15 @@ +use codeupipe::{Payload, StepError, Tap, Value}; + +use super::sink; + +/// append get(key) to the runner's tap log +pub struct RecordTap { + pub key: String, +} + +impl Tap for RecordTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + sink::record(payload.get(&self.key).cloned().unwrap_or(Value::Null)); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/remove_filter.rs b/conformance/runners/rust/src/cup_ops/remove_filter.rs new file mode 100644 index 0000000..6678655 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/remove_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// remove(key) +pub struct RemoveFilter { + pub key: String, +} + +impl Filter for RemoveFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.remove(&self.key); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/run_again_filter.rs b/conformance/runners/rust/src/cup_ops/run_again_filter.rs new file mode 100644 index 0000000..b149719 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/run_again_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Map, Payload, StepError, Value}; + +use super::sink; + +/// when get("inner") is not true: run the pipeline that this step is in one +/// more time, with `input` and a new State. How a pipeline is run again +/// depends on the target, so the runner gives that as a function +/// (`sink::run_again`). Then the payload goes on with no change. +pub struct RunAgainFilter { + pub input: Map, +} + +impl Filter for RunAgainFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("inner") != Some(&Value::Bool(true)) { + sink::run_again(self.input.clone())?; + } + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/set_filter.rs b/conformance/runners/rust/src/cup_ops/set_filter.rs new file mode 100644 index 0000000..f75b1d0 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/set_filter.rs @@ -0,0 +1,14 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = value +pub struct SetFilter { + pub key: String, + pub value: Value, +} + +impl Filter for SetFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert(self.key.as_str(), self.value.clone()); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/sink.rs b/conformance/runners/rust/src/cup_ops/sink.rs new file mode 100644 index 0000000..eb1f6d2 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/sink.rs @@ -0,0 +1,49 @@ +//! Where the steps of the runner record what they see, and the hook for +//! running the pipeline of a case again. The runner empties it before each +//! case. +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Map, State, StepError, Value}; + +/// Runs the pipeline of the case again, with this input and a new State. +pub type RunAgain = Rc Result<(), StepError>>; + +thread_local! { + static TAPS: RefCell> = const { RefCell::new(Vec::new()) }; + static RUN_AGAIN: RefCell> = const { RefCell::new(None) }; + static INNER_STATES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +pub fn record(value: Value) { + TAPS.with(|taps| taps.borrow_mut().push(value)); +} + +/// What was recorded since the last call, and an empty log. +pub fn take() -> Vec { + TAPS.with(|taps| taps.take()) +} + +/// How the `run_again` step of this case runs its pipeline again. +pub fn set_run_again(run_again: Option) { + RUN_AGAIN.with(|slot| slot.replace(run_again)); +} + +pub fn run_again(input: Map) -> Result<(), StepError> { + // Taken out of the cell first: the run that it starts reads the cell too. + let run_again = RUN_AGAIN.with(|slot| slot.borrow().clone()); + match run_again { + Some(run_again) => run_again(input), + None => Err("no pipeline to run again".into()), + } +} + +/// Keep the State of a run that a `run_again` step started. +pub fn keep_inner_state(state: State) { + INNER_STATES.with(|states| states.borrow_mut().push(state)); +} + +/// The States kept since the last call, and an empty list. +pub fn take_inner_states() -> Vec { + INNER_STATES.with(|states| states.take()) +} diff --git a/conformance/runners/rust/src/cup_ops/truthy_predicate.rs b/conformance/runners/rust/src/cup_ops/truthy_predicate.rs new file mode 100644 index 0000000..3ac1275 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/truthy_predicate.rs @@ -0,0 +1,20 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// get(key) is truthy, as in Python: null, false, 0, "" and an empty list +/// or map are false. A missing key is false. +pub struct TruthyPredicate { + pub key: String, +} + +impl Predicate for TruthyPredicate { + fn test(&self, payload: &Payload) -> bool { + match payload.get(&self.key) { + None | Some(Value::Null) => false, + Some(Value::Bool(b)) => *b, + Some(Value::Number(n)) => *n != 0.0, + Some(Value::String(s)) => !s.is_empty(), + Some(Value::List(items)) => !items.is_empty(), + Some(Value::Map(map)) => !map.is_empty(), + } + } +} diff --git a/conformance/runners/rust/src/json.rs b/conformance/runners/rust/src/json.rs new file mode 100644 index 0000000..ca7f163 --- /dev/null +++ b/conformance/runners/rust/src/json.rs @@ -0,0 +1,337 @@ +//! JSON text to a codeupipe `Value` and back. A runner must not download +//! anything, so the runner has its own small reader and writer. + +use codeupipe::{Map, Value}; + +/// Read one JSON value from `text`. +pub fn read(text: &str) -> Result { + let mut reader = Reader { text, at: 0 }; + let value = reader.value()?; + reader.space(); + if reader.at != text.len() { + return Err(reader.problem("text after the value")); + } + Ok(value) +} + +/// Write `value` as JSON text. +pub fn write(value: &Value) -> String { + let mut out = String::new(); + write_into(value, &mut out); + out +} + +fn write_into(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + // Display gives 12 for 12.0. JSON has no NaN or infinity: null, as JavaScript writes them. + Value::Number(n) if n.is_finite() => out.push_str(&n.to_string()), + Value::Number(_) => out.push_str("null"), + Value::String(s) => write_string(s, out), + Value::List(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(','); + } + write_into(item, out); + } + out.push(']'); + } + Value::Map(map) => { + out.push('{'); + for (index, (key, item)) in map.iter().enumerate() { + if index > 0 { + out.push(','); + } + write_string(key, out); + out.push(':'); + write_into(item, out); + } + out.push('}'); + } + } +} + +fn write_string(s: &str, out: &mut String) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); +} + +struct Reader<'a> { + text: &'a str, + at: usize, +} + +impl Reader<'_> { + fn problem(&self, what: &str) -> String { + format!("not JSON at byte {}: {what}", self.at) + } + + fn peek(&self) -> Option { + self.text.as_bytes().get(self.at).copied() + } + + fn space(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + + fn expect(&mut self, byte: u8) -> Result<(), String> { + self.space(); + if self.peek() == Some(byte) { + self.at += 1; + Ok(()) + } else { + Err(self.problem(&format!("expected '{}'", byte as char))) + } + } + + fn word(&mut self, word: &str, value: Value) -> Result { + if self.text[self.at..].starts_with(word) { + self.at += word.len(); + Ok(value) + } else { + Err(self.problem("unknown word")) + } + } + + fn value(&mut self) -> Result { + self.space(); + match self.peek() { + Some(b'n') => self.word("null", Value::Null), + Some(b't') => self.word("true", Value::Bool(true)), + Some(b'f') => self.word("false", Value::Bool(false)), + Some(b'"') => self.string().map(Value::String), + Some(b'[') => self.list(), + Some(b'{') => self.map(), + Some(b'-' | b'0'..=b'9') => self.number(), + Some(_) => Err(self.problem("unexpected character")), + None => Err(self.problem("end of the text")), + } + } + + fn number(&mut self) -> Result { + let start = self.at; + while matches!( + self.peek(), + Some(b'-' | b'+' | b'.' | b'e' | b'E' | b'0'..=b'9') + ) { + self.at += 1; + } + self.text[start..self.at] + .parse::() + .map(Value::Number) + .map_err(|_| self.problem("bad number")) + } + + fn list(&mut self) -> Result { + self.expect(b'[')?; + let mut items = Vec::new(); + self.space(); + if self.peek() == Some(b']') { + self.at += 1; + return Ok(Value::List(items)); + } + loop { + items.push(self.value()?); + self.space(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b']') => { + self.at += 1; + return Ok(Value::List(items)); + } + _ => return Err(self.problem("expected ',' or ']'")), + } + } + } + + fn map(&mut self) -> Result { + self.expect(b'{')?; + let mut map = Map::new(); + self.space(); + if self.peek() == Some(b'}') { + self.at += 1; + return Ok(Value::Map(map)); + } + loop { + self.space(); + if self.peek() != Some(b'"') { + return Err(self.problem("expected a key")); + } + let key = self.string()?; + self.expect(b':')?; + let value = self.value()?; + map.insert(key, value); + self.space(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b'}') => { + self.at += 1; + return Ok(Value::Map(map)); + } + _ => return Err(self.problem("expected ',' or '}'")), + } + } + } + + fn string(&mut self) -> Result { + self.at += 1; // the opening quote + let mut out = String::new(); + loop { + let rest = &self.text[self.at..]; + let Some(c) = rest.chars().next() else { + return Err(self.problem("string with no end")); + }; + self.at += c.len_utf8(); + match c { + '"' => return Ok(out), + '\\' => out.push(self.escape()?), + c => out.push(c), + } + } + } + + fn escape(&mut self) -> Result { + let Some(byte) = self.peek() else { + return Err(self.problem("string with no end")); + }; + self.at += 1; + Ok(match byte { + b'"' => '"', + b'\\' => '\\', + b'/' => '/', + b'b' => '\u{8}', + b'f' => '\u{c}', + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'u' => { + let first = self.hex4()?; + if (0xd800..0xdc00).contains(&first) { + // A pair of UTF-16 halves: 😀 + if !self.text[self.at..].starts_with("\\u") { + return Err(self.problem("half of a UTF-16 pair")); + } + self.at += 2; + let second = self.hex4()?; + let code = 0x10000 + ((first - 0xd800) << 10) + (second.wrapping_sub(0xdc00)); + char::from_u32(code).ok_or_else(|| self.problem("bad UTF-16 pair"))? + } else { + char::from_u32(first).ok_or_else(|| self.problem("bad \\u escape"))? + } + } + _ => return Err(self.problem("unknown escape")), + }) + } + + fn hex4(&mut self) -> Result { + let digits = self + .text + .get(self.at..self.at + 4) + .ok_or_else(|| self.problem("short \\u escape"))?; + let code = u32::from_str_radix(digits, 16).map_err(|_| self.problem("bad \\u escape"))?; + self.at += 4; + Ok(code) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use codeupipe::Map; + + #[test] + fn reads_each_kind_of_value() { + assert_eq!(read("null").unwrap(), Value::Null); + assert_eq!(read("true").unwrap(), Value::Bool(true)); + assert_eq!(read("false").unwrap(), Value::Bool(false)); + assert_eq!(read("12").unwrap(), Value::Number(12.0)); + assert_eq!(read("-1.5e2").unwrap(), Value::Number(-150.0)); + assert_eq!(read("\"x\"").unwrap(), Value::from("x")); + assert_eq!( + read(" [1, \"two\", [3]] ").unwrap(), + Value::List(vec![ + Value::from(1), + Value::from("two"), + Value::List(vec![Value::from(3)]) + ]) + ); + assert_eq!( + read("{\"a\": 1, \"b\": {}}").unwrap(), + Value::Map(Map::from([ + ("a", Value::from(1)), + ("b", Value::Map(Map::new())) + ])) + ); + } + + #[test] + fn reads_the_escapes_of_a_string() { + assert_eq!( + read(r#""q\" b\\ s\/ n\n t\t ué pair😀""#).unwrap(), + Value::from("q\" b\\ s/ n\n t\t u\u{e9} pair\u{1f600}") + ); + } + + #[test] + fn keeps_the_order_of_the_keys() { + let value = read("{\"z\": 1, \"a\": 2, \"m\": 3}").unwrap(); + let keys: Vec<&str> = value.as_map().unwrap().keys().collect(); + assert_eq!(keys, ["z", "a", "m"]); + } + + #[test] + fn refuses_text_that_is_not_json() { + for text in ["", "[1,", "{\"a\" 1}", "tru", "\"open", "1 2", "{1: 2}"] { + assert!(read(text).is_err(), "{text:?}"); + } + } + + #[test] + fn writes_each_kind_of_value() { + let value = Value::Map(Map::from([ + ("n", Value::from(1.5)), + ("w", Value::from(12)), + ("s", Value::from("x")), + ("b", Value::from(false)), + ("z", Value::Null), + ( + "l", + Value::List(vec![Value::from(1), Value::List(vec![Value::from(3)])]), + ), + ])); + assert_eq!( + write(&value), + r#"{"n":1.5,"w":12,"s":"x","b":false,"z":null,"l":[1,[3]]}"# + ); + } + + #[test] + fn writes_the_escapes_of_a_string() { + assert_eq!( + write(&Value::from("q\" b\\ n\n c\u{1} \u{e9}")), + r#""q\" b\\ n\n c\u0001 é""# + ); + } + + #[test] + fn what_it_writes_reads_back_the_same() { + let text = r#"{"v":{"n":1.5,"s":"x","b":false,"z":null,"l":[1,"two",[3]]},"e":"a\"b"}"#; + assert_eq!(write(&read(text).unwrap()), text); + } +} diff --git a/conformance/runners/rust/src/main.rs b/conformance/runners/rust/src/main.rs new file mode 100644 index 0000000..4e390b5 --- /dev/null +++ b/conformance/runners/rust/src/main.rs @@ -0,0 +1,352 @@ +//! Rust conformance runner: runs the cases and reports what happened. +//! +//! runner < cases.json > results.json +//! +//! Targets +//! run the Pipeline of packages/codeupipe-rs +//! bench:run the same, timed on the scenarios of conformance/bench +//! +//! The runner never judges; conformance/run.py compares the results of every +//! target with the same expectations. +mod bench; +mod cup_ops; +mod json; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::io::{self, Read, Write}; +use std::process::ExitCode; +use std::rc::Rc; + +use codeupipe::{Filter, Map, Payload, Pipeline, Predicate, State, Value}; +use cup_ops::{ + sink, AddFilter, AppendFilter, CopyFilter, EqualsPredicate, FailFilter, FailTap, HasFilter, + MergeFilter, MutateFilter, NestedPipeline, NoopHook, NoopTap, RecordHook, RecordTap, + RemoveFilter, RunAgainFilter, SetFilter, TruthyPredicate, +}; + +/// A case that this runner cannot express. It is reported, never judged. +struct Unsupported(String); + +fn field<'a>(spec: &'a Map, name: &str) -> Result<&'a Value, Unsupported> { + spec.get(name) + .ok_or_else(|| Unsupported(format!("the step has no '{name}'"))) +} + +fn text(spec: &Map, name: &str) -> Result { + field(spec, name)? + .as_str() + .map(str::to_string) + .ok_or_else(|| Unsupported(format!("'{name}' is not a string"))) +} + +fn number(spec: &Map, name: &str) -> Result { + field(spec, name)? + .as_f64() + .ok_or_else(|| Unsupported(format!("'{name}' is not a number"))) +} + +fn map<'a>(spec: &'a Map, name: &str) -> Result<&'a Map, Unsupported> { + field(spec, name)? + .as_map() + .ok_or_else(|| Unsupported(format!("'{name}' is not an object"))) +} + +fn value(spec: &Map, name: &str) -> Result { + field(spec, name).cloned() +} + +/// Where a Filter goes: alone, or behind a Predicate. `add_filter` and +/// `add_valve` take the type of the step, which gives its label, so the +/// Filter is handed over by its own type and not as a `Box`. +trait UseFilter { + fn add(self, filter: F); +} + +struct AsFilter<'a>(&'a mut Pipeline); + +impl UseFilter for AsFilter<'_> { + fn add(self, filter: F) { + self.0.add_filter(filter); + } +} + +struct AsValve<'a, P>(&'a mut Pipeline, P); + +impl UseFilter for AsValve<'_, P> { + fn add(self, filter: F) { + self.0.add_valve(self.1, filter); + } +} + +fn add_filter(spec: &Map, to: impl UseFilter) -> Result<(), Unsupported> { + match text(spec, "op")?.as_str() { + "set" => to.add(SetFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "copy" => to.add(CopyFilter { + from: text(spec, "from")?, + to: text(spec, "to")?, + }), + "add" => to.add(AddFilter { + key: text(spec, "key")?, + by: number(spec, "by")?, + }), + "append" => to.add(AppendFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "remove" => to.add(RemoveFilter { + key: text(spec, "key")?, + }), + "merge" => to.add(MergeFilter { + value: map(spec, "value")?.clone(), + }), + "has" => to.add(HasFilter { + key: text(spec, "key")?, + to: text(spec, "to")?, + }), + "mutate" => to.add(MutateFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "fail" => to.add(FailFilter { + message: text(spec, "message")?, + }), + "run_again" => to.add(RunAgainFilter { + input: map(spec, "input")?.clone(), + }), + op => return Err(Unsupported(format!("unknown filter op '{op}'"))), + } + Ok(()) +} + +fn add_tap(spec: &Map, pipeline: &mut Pipeline) -> Result<(), Unsupported> { + match text(spec, "op")?.as_str() { + "record" => pipeline.add_tap(RecordTap { + key: text(spec, "key")?, + }), + "fail" => pipeline.add_tap(FailTap { + message: text(spec, "message")?, + }), + "noop" => pipeline.add_tap(NoopTap), + op => return Err(Unsupported(format!("unknown tap op '{op}'"))), + } + Ok(()) +} + +fn add_valve(when: &Map, filter: &Map, pipeline: &mut Pipeline) -> Result<(), Unsupported> { + match text(when, "op")?.as_str() { + "equals" => { + let predicate = EqualsPredicate { + key: text(when, "key")?, + value: value(when, "value")?, + }; + add_filter(filter, AsValve(pipeline, predicate)) + } + "truthy" => { + let predicate = TruthyPredicate { + key: text(when, "key")?, + }; + add_filter(filter, AsValve(pipeline, predicate)) + } + op => Err(Unsupported(format!("unknown valve op '{op}'"))), + } +} + +fn steps_of(spec: &Map) -> Result<&[Value], Unsupported> { + field(spec, "steps")? + .as_list() + .ok_or_else(|| Unsupported("'steps' is not a list".to_string())) +} + +fn build(steps: &[Value], pipeline: &mut Pipeline) -> Result<(), Unsupported> { + for step in steps { + let step = step + .as_map() + .ok_or_else(|| Unsupported("a step is not an object".to_string()))?; + match text(step, "kind")?.as_str() { + "filter" => add_filter(step, AsFilter(pipeline))?, + "tap" => add_tap(step, pipeline)?, + "valve" => add_valve(map(step, "when")?, map(step, "filter")?, pipeline)?, + "pipeline" => { + let mut inner = Pipeline::named(NestedPipeline); + build(steps_of(step)?, &mut inner)?; + pipeline.add_pipeline(inner); + } + kind => return Err(Unsupported(format!("unknown step kind '{kind}'"))), + } + } + Ok(()) +} + +/// Label to the step name of the case. The cases name their steps, and the +/// runtime reports labels (conformance/API.md). The steps are added in the +/// order of the case, so the two lists are side by side. A label that the +/// runtime reports and that is not in this map stays as it is, and fails the +/// comparison. +fn names_by_label(pipeline: &Pipeline, steps: &[Value]) -> HashMap { + pipeline + .labels() + .into_iter() + .zip(steps) + .map(|(label, step)| { + let name = step.as_map().and_then(|step| step.get("name")); + ( + label, + name.and_then(Value::as_str).unwrap_or("").to_string(), + ) + }) + .collect() +} + +fn names(labels: &[String], named: &impl Fn(&str) -> String) -> Value { + Value::List( + labels + .iter() + .map(|label| Value::from(named(label))) + .collect(), + ) +} + +fn run_case(spec: &Map) -> Result { + let steps = steps_of(spec)?; + let mut pipeline = Pipeline::new(); + build(steps, &mut pipeline)?; + let by_label = names_by_label(&pipeline, steps); + let named = |label: &str| { + by_label + .get(label) + .cloned() + .unwrap_or_else(|| label.to_string()) + }; + let hooks = spec.get("hooks") == Some(&Value::Bool(true)); + let events = Rc::new(RefCell::new(Vec::new())); + if hooks { + pipeline.use_hook(RecordHook { + events: Rc::clone(&events), + }); + } + + // A run_again step runs this pipeline again, as the TypeScript runner does. + let pipeline = Rc::new(pipeline); + let again = Rc::downgrade(&pipeline); + sink::set_run_again(Some(Rc::new(move |input: Map| { + let pipeline = again.upgrade().ok_or("the pipeline is gone")?; + let mut state = State::new(); + let result = pipeline.run_with_state(&Payload::from_map(input), &mut state); + sink::keep_inner_state(state); + result.map(|_| ()) + }))); + + let input = Payload::from_map(map(spec, "input")?.clone()); + let mut result = Map::new(); + result.insert("id", value(spec, "id")?); + // With "given_state", the caller gives the State of the run. If not, the run makes one. + let mut given = (spec.get("given_state") == Some(&Value::Bool(true))).then(State::new); + let ran = match given.as_mut() { + Some(state) => pipeline.run_with_state(&input, state), + None => pipeline.run(&input), + }; + match ran { + Ok(output) => result.insert("output", output.to_dict()), + Err(error) => result.insert("error", error.to_string()), + }; + sink::set_run_again(None); + let report = |state: &State| { + let errors: Vec = state + .errors() + .iter() + .map(|(label, _)| label.clone()) + .collect(); + Value::Map(Map::from([ + ("executed", names(state.executed(), &named)), + ("skipped", names(state.skipped(), &named)), + ("errors", names(&errors, &named)), + ])) + }; + let last = pipeline.state(); + result.insert("state", report(given.as_ref().unwrap_or(&last))); + result.insert("pipeline_state", report(&last)); + let inner: Vec = sink::take_inner_states().iter().map(report).collect(); + result.insert("inner_states", Value::List(inner)); + result.insert("taps", sink::take()); + if hooks { + let events: Vec = events + .borrow() + .iter() + .map(|event| { + let (kind, label) = event.split_once(':').unwrap_or((event, "")); + let step = if label.is_empty() { + String::new() + } else { + named(label) + }; + Value::from(format!("{kind}:{step}")) + }) + .collect(); + result.insert("hooks", Value::List(events)); + } + result.insert("input_after", input.to_dict()); + Ok(Value::Map(result)) +} + +fn bench_case(spec: &Map) -> Result { + let mut pipeline = Pipeline::new(); + build(steps_of(spec)?, &mut pipeline)?; + if spec.get("hooks") == Some(&Value::from("noop")) { + pipeline.use_hook(NoopHook); + } + let input = Payload::from_map(map(spec, "input")?.clone()); + Ok(bench::bench(value(spec, "id")?, pipeline, input)) +} + +fn main() -> ExitCode { + let target = std::env::args().nth(1).unwrap_or_default(); + let run: fn(&Map) -> Result = match target.as_str() { + "run" => run_case, + "bench:run" => bench_case, + _ => { + eprintln!("unknown target '{target}'. The targets are: run, bench:run"); + return ExitCode::from(2); + } + }; + let mut text = String::new(); + if let Err(error) = io::stdin().read_to_string(&mut text) { + eprintln!("cannot read the cases: {error}"); + return ExitCode::from(2); + } + let cases = match json::read(&text) { + Ok(Value::List(cases)) => cases, + Ok(_) => { + eprintln!("the cases are not a list"); + return ExitCode::from(2); + } + Err(error) => { + eprintln!("cannot read the cases: {error}"); + return ExitCode::from(2); + } + }; + + let mut results = Vec::new(); + for case in &cases { + sink::take(); + let Some(spec) = case.as_map() else { + eprintln!("a case is not an object"); + return ExitCode::from(2); + }; + results.push(match run(spec) { + Ok(result) => result, + Err(Unsupported(reason)) => Value::Map(Map::from([ + ("id", spec.get("id").cloned().unwrap_or(Value::Null)), + ("unsupported", Value::from(reason)), + ])), + }); + } + let mut out = io::stdout().lock(); + if writeln!(out, "{}", json::write(&Value::List(results))).is_err() { + return ExitCode::from(2); + } + ExitCode::SUCCESS +} diff --git a/docs/compile.md b/docs/compile.md index a572878..a0fc93d 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -73,6 +73,7 @@ generated `run()` (scorecard findings F3–F8). | Literal instance settings (`self.key` holding a str, number, ...) | compiled in as constants | `test_export_instance_config` | | Type hints on a step (`Payload[Reads] -> Payload[Writes]`, hinted locals) | erased: compiled code carries no hints and never imports their names | `test_export_typed_steps` | | Payload ops `get insert remove merge has to_dict copy` | dict operations; a write is an in-place statement (`data[k] = v`) | `test_rewrite_payload_ops` | +| The Payload parameter under any name (`call(self, order)`, `test(self, p)`) | the same, followed by its name: an unrelated local called `payload` stays separate | `test_export_parameter_name` | Evidence: all 30 conformance cases on both layouts (`python:static-steps`, `python:static-inline`), and the parity spike at 2400/2400 inputs. @@ -90,7 +91,7 @@ and the older shape tests beside it. | `observe=True` with `lowering="inline"` (`ValueError`) | inline writes the payload in place, so a Hook holding it would see later steps' writes | use the default `steps` layout | | **Async steps** (async `call`/`observe`, Valve around one, async `test`) | static `run` is sync, as the dynamic `run` is: it raises `AsyncStepInSyncRun` on the same pipeline | make the step sync | | **`payload.trace_id`, `payload.lineage`, `payload.with_trace`** | a plain dict carries neither | keep the step dynamic | -| any other payload attribute (`payload._data`, ...) | same: only the operations above have a dict form | use an compilable operation | +| any other attribute of the Payload parameter (`payload._data`, `order.price` on a type of your own, ...) | same: only the operations above have a dict form. The refusal names the parameter as the step wrote it | use a compilable operation, or keep a step over a type of your own dynamic | | callables or objects on `self` (filter or predicate), steps writing `self.x` | no source or literal to compile in | literal settings (predicates and filters are module-level classes anyway: the runtime refuses anything else) | ## What the fast mode gives up diff --git a/docs/rust-plan.md b/docs/rust-plan.md new file mode 100644 index 0000000..788d399 --- /dev/null +++ b/docs/rust-plan.md @@ -0,0 +1,511 @@ +# Plan: codeupipe for Rust + +This is a plan, not a reference. It says how the third runtime is added: a +Rust runtime with the same contract ([conformance/API.md](../conformance/API.md)), +judged by the same conformance cases, in the same layout as TypeScript. + +**How it is built: from the contract, tests first.** Each behaviour starts as +a failing test that comes from a rule of the contract or from a test of the +TypeScript package. **When in doubt, do what the TypeScript package does**: it +is the reference for every question that the contract does not answer. + +The Rust runtime in the old repository (`runtimes/rs`) is not the start. Its +API is from before version 2: steps named by hand, no Predicate, no labels. +It can be read for ideas. No code is copied from it. + +## Versions + +The Rust packages start at **0.1.0**, as the TypeScript packages did. A version +below 1.0 is not stable yet, and a new version can change in a way that is not +compatible. For Cargo that means: 0.1.x to 0.2.0 is a change that is not +compatible. + +The packages are not on crates.io at first. Each has `publish = false`, as each +TypeScript package has `"private": true`. Until they are published, a project +uses them from this repository: + +```toml +[dependencies] +codeupipe = { git = "https://github.com/orchestrate-solutions/codeupipe", branch = "beta" } +``` + +Cargo finds the crate by its name inside the repository, so this works with +the crate in `packages/codeupipe-rs`. + +## Layout + +The packages are beside the others, with `-rs` where TypeScript has `-ts`. + +| Path | Crate | Contents | Phase | +|---|---|---|---| +| `packages/codeupipe-rs/` | `codeupipe` | The eight primitives. No dependencies | 1 | +| `conformance/runners/rust/` | (binary, not published) | The conformance runner, with its `cup_ops` steps | 2 | +| `packages/codeupipe-compile-rs/` | `codeupipe-compile` | Writes a pipeline as plain Rust with no use of codeupipe | 3 | +| `packages/codeupipe-lint-rs/` | `codeupipe-lint` | The linter for rules that the type system does not already check | 4 | + +A root `Cargo.toml` makes the Rust crates one Cargo workspace. It is not a +package, as the root `package.json` is not a package. `Cargo.lock` is +committed, since the workspace has a binary (the runner). `target/` goes in +`.gitignore`. + +One primitive is in each file, and the file name is the type name in +snake_case, as in Python: `DiscountFilter` is in `discount_filter.rs`. + +### The core crate + +``` +packages/codeupipe-rs/ + Cargo.toml name = "codeupipe", version = "0.1.0", publish = false + README.md LICENSE NOTICE + src/ + lib.rs the public names, as index.ts has them + value.rs Value: null, bool, number, string, list, map + payload.rs Payload + filter.rs trait Filter + predicate.rs trait Predicate + tap.rs trait Tap + hook.rs trait Hook, every method with a default that does nothing + state.rs State + valve.rs Valve (not a public name) + pipeline.rs Pipeline + named_pipeline.rs trait NamedPipeline (see decision D1) + step_label.rs the label from the type name + step_error.rs the error type of a step (see decision D4) + tests/ + payload.rs pipeline.rs step_label.rs pricing_example.rs + examples/pricing/ + main.rs discount_filter.rs mark_bulk_filter.rs at_least_predicate.rs + audit_tap.rs print_steps_hook.rs pricing_pipeline.rs +``` + +## Decisions + +These are the questions that Rust raises and Python and TypeScript do not. +**They are settled as recommended** (2026-10-27). + +### D1. A named pipeline + +**The question.** The contract says to make a nested pipeline a named type +(`class UserOnboarding(Pipeline)`), so that its label says what it is. Rust has +no classes to extend. + +**Recommendation.** A trait that a named type implements. Its one method adds +the steps. The label is the name of the type. + +```rust +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} + +let mut pipeline = Pipeline::named(PricingPipeline); +``` + +`add_steps` is also what the compiler reads (phase 3), as the TypeScript +compiler reads the constructor. A plain `Pipeline::new()` has the label +`Pipeline`, as in the contract. + +### D2. How a write returns the Payload + +**The question.** The contract says that a write changes the Payload and +returns it, so that `return payload.insert(k, v)` is one line. + +**Settled.** A Filter gets `&mut Payload` and writes in place. A write takes +`&mut self` and returns it, so writes chain: +`payload.insert("a", 1).remove("b")`. + +```rust +fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0); + payload.insert("price", (price * (1.0 - self.rate) * 100.0).round() / 100.0); + Ok(()) +} +``` + +The first recommendation was by value: `call(payload: Payload) -> Result`. +It was changed in phase 1, against the TypeScript package. TypeScript changes +the Payload in place, and its `onError` gets that same Payload, with the +writes the failing step made before it threw. By value, the Payload is gone +when the Filter fails, and `on_error` has none to give. `&mut` does what +TypeScript does, and a Filter cannot return a different Payload (which the +TypeScript compiler has to refuse). + +A Tap, a Predicate and a Hook get `&Payload`. So **only a Filter can write, and +the compiler checks it**: lint rule `CUP-I004` (a write outside a Filter) is a +type error in Rust, not a lint finding. + +### D3. `get` with a default + +**The question.** The contract has `get(k, default)`. Rust has no default +arguments and no overloads. + +**Recommendation.** `get(k) -> Option<&Value>`. A default is +`.unwrap_or(...)`, which is the language's own way, not a second name. +`Value` has `as_f64`, `as_str`, `as_bool`, `as_list`, `as_map`. Absent is +`None`. + +### D4. Errors + +**The question.** A Filter failure raises, the Hook gets "the error itself", +and State keeps (label, error). + +**Recommendation.** A step returns `Result`. `StepError` is +`Box`. `on_error` gets `&StepError`, and +then `run` returns the same error. A Tap error goes into `state.errors`. + +A **panic is not a failure**. It is not caught: a panicking Tap stops the run. +The README says so. Catching panics (`catch_unwind`) would make the Tap rule +hold for panics too, but it needs `UnwindSafe` bounds on every step. It can +be added later without changing the API. + +### D5. Labels + +**The question.** The label is the name of the step's type. + +**Recommendation.** `add_filter(&mut self, filter: F)` +reads `std::any::type_name::()` when the step is added, and keeps the last +part of the path (`examples::pricing::DiscountFilter` gives `DiscountFilter`). +Generic arguments are dropped (`Wrapper` gives `Wrapper`). A closure +does not implement `Filter`, so it cannot be a step. There is no blanket +implementation for `Fn`. + +What the contract refuses with a `TypeError` when a step is added is a +compile error in Rust: a value without `call` does not implement `Filter`, and +`add_valve(filter, predicate)` does not type-check. Tests for these use +`compile_fail` doc tests, which need no dependency. + +### D6. Values and the order of keys + +**The question.** Values are JSON-shaped, and the core crate has no +dependencies, so there is no `serde_json::Value`. + +**Recommendation.** An own `Value` enum: `Null`, `Bool`, `Number(f64)`, +`String`, `List(Vec)`, `Map(...)`. One number type, as JavaScript has, +and as the conformance judge compares (`1 == 1.0`). + +Keys keep the order in which they were inserted, as a Python dict and a +JavaScript Map do. `to_dict` gives them in that order. The judge does not +compare order, so this is for people, not for conformance. + +`copy()` is a deep copy in this version. The contract says the two Payloads +"share their values", but in Rust a shared value cannot be changed without a +`Cell` or a lock, so a deep copy shows the same behaviour. Sharing with `Rc` +can be added later for speed, without a change to the API. + +### D7. Sync and async + +**Recommendation.** 0.1.0 has `run` only, and it is sync (contract principle +3). A sync caller never pays for an executor. `run_async` waits for a +conformance case that needs it, and for a choice of how to stay free of +dependencies (an `async fn` in the traits, driven by the caller's executor). + +### D8. State and the run + +**Changed (2026-10-27): `run(&self, ...)`**, see "A State for each run" +below. The first recommendation was: + +`run(&mut self, input: &Payload) -> Result`. +It copies `input`, so the caller's Payload is not changed (contract), and it +writes a new `State` into the pipeline. `pipeline.state()` reads it. +`&mut self` means that two runs of one pipeline at the same time are not +possible. That is simpler than the TypeScript rule ("the run that started +last") and it needs no lock. + +`timings` keeps a `Duration` for each label. `observe(timing, lineage)` takes +two `bool`s. + +On `wasm32-unknown-unknown`, `std::time::Instant` panics. Timing is off by +default, so a pipeline runs in WebAssembly. Timing there is out of scope for +0.1.0, and the README says so. + +### D9. Edition and Rust version + +**Recommendation.** Edition 2021, `rust-version = "1.75"`. The core uses +nothing newer. This machine has 1.89. + +### D10. The crate names + +`codeupipe`, `codeupipe-compile` and `codeupipe-lint` are the names in the +other languages. Before the first publish, check that they are free on +crates.io. The old repository used `codeupipe-core`. + +### D11. A payload of your own + +**Settled (2026-10-27).** `Payload` stays the default: a map that any +developer can read, which shows that anything can go there. A developer, or +an agent, that would write a struct anyway can opt in: each primitive takes +the data type as a parameter with `Payload` as its default (`Filter

`, `Pipeline

`), and the type implements `PipelineData`: +`Clone` (for the copy that `run` works on) and an optional `stamp` for +lineage. Code written for `Payload` does not change; `Pipeline::new()` infers +the type from its steps. + +Python and TypeScript have the same option (2026-10-27): Python takes any +object with `copy()`, and TypeScript has the same type parameter. In all +three, lineage is kept only by a type that can keep it, and does nothing for +any other ("A type of your own" in conformance/API.md). A step for one +type in a pipeline of another is a compile error, checked by a doc test. + +Measured (release, Apple Silicon): the pricing pipeline runs in 0.024 µs +with an `Order` struct against 0.18 µs with a Payload, and 50 writes take +0.46 µs against 4.3 µs. What is left, about 9 ns a step, is the pipeline's own +work: State, Hooks and the call through `dyn Filter`. + +## Phase 1: the core crate + +**Done** (2026-10-27): `packages/codeupipe-rs`, 47 tests (16 Payload, 17 +Pipeline, 7 labels, 3 example, 4 doc tests, 3 of them `compile_fail`), +`cargo clippy -- -D warnings` and `cargo fmt --check` clean. The tests are +ported from `packages/codeupipe-ts/tests`, except those that have no Rust form: +async steps, two runs at the same time, a class with no name, a key named like +an Object property. + +The eight primitives, to the contract, with the decisions above. + +Steps: + +1. `value.rs` and `payload.rs`, with tests first (RED, GREEN, REFACTOR), from + the Payload table of the contract. +2. The traits: `Filter`, `Predicate`, `Tap`, `Hook`. `step_label.rs`. +3. `State`, `Valve`, `Pipeline` (`add_filter`, `add_valve`, `add_tap`, + `add_pipeline`, `use_hook`, `run`, `state`, `labels`, `observe`), and + `NamedPipeline`. +4. The pricing example, with its test, giving + `{"price": 90.0, "qty": 12.0, "bulk": true}`. +5. The README, in the form of the TypeScript README: the primitives, the + example, the names, the differences from Python. + +Done when: `cargo test`, `cargo clippy -- -D warnings` and `cargo fmt --check` +pass, and `cargo run --example pricing` prints the result and the State. + +## Phase 2: conformance + +**Done** (2026-10-27): `rust:run` passed all 27 cases, with nothing +unsupported. The contract has since added three cases, from "A State +belongs to one run" (`run(p, state)`, and a run inside a run); Rust does not +pass them yet; "A State for each run" below adds them. To check that the runner catches a broken runtime, the core was +changed so that it did not record a Tap failure: 3 cases failed. The runner +has 7 tests of its JSON reader and writer. `bench.py` measures `rust:run`, its +baseline is kept under its own key (`rust`, not a Python version), and CI has a `rust` job, which also +builds with Rust 1.75 (D9). + +One thing that only Rust needed: `add_filter` and `add_valve` take the type +of the step, which gives its label. A step that the runner picks at run time +cannot be a `Box` (its label would be `Box`), so the runner hands +each Filter over by its own type through a small generic trait (`UseFilter`). + +The first numbers showed `rust:run` 4 to 5 times faster than `python:run` +but about 2 times **slower** than compiled Python. The time was in +allocation, not in dispatch (50 no-op steps take 0.42 µs). Three changes to +the core, with no change to its behaviour or its API: + +1. **`Map::insert` looks the key up first**, so a key that is there makes no + new `String`. +2. **State keeps its storage between runs** and writes the labels over, so a + run like the last one makes no new label `String`. +3. **A key is an `Arc`, made once**, shared by the index and by every + copy of the map. A map of 8 keys or fewer is searched in order and has no + hash index; a larger one gets an index with room to grow. The hash stays + std's SipHash: a faster hasher with no random seed would let crafted keys + slow down a map that is read from untrusted JSON. + +µs per run, Apple Silicon, one run: + +| scenario | python:run | python:static-inline | rust:run before | rust:run now | +|---|---|---|---|---| +| `micro-pass` (10 filters) | 4.94 | 0.606 | 1.10 | 0.246 | +| `valve-gated` | 5.81 | 0.801 | 1.24 | 0.301 | +| `taps` | 5.34 | 0.602 | 1.17 | 0.256 | +| `hook` | 5.88 | — | 1.10 | 0.247 | +| `heavy-allocation` (50 inserts) | 16.05 | 1.49 | 7.26 | 4.24 | + +The dynamic Rust run is now 20 times faster than `python:run`, and 2.5 times +faster than compiled Python, except where a run makes 50 new keys: there each +new key is an allocation and a hash, and a Python dict is hard to beat. **So +phase 3 waits**: the dynamic run already costs about 25 ns a step, and a +compiler would remove nanoseconds that few pipelines would notice. The README +says "no compiler yet". + +A runner in `conformance/runners/rust/`, with the protocol of the others: +`runner < cases.json > results.json`. It never judges. + +- **Steps.** `cup_ops/` has one file for each step type, with the names that + the TypeScript runner uses: `SetFilter`, `CopyFilter`, `AddFilter`, + `AppendFilter`, `RemoveFilter`, `MergeFilter`, `HasFilter`, `MutateFilter`, + `FailFilter`, `RecordTap`, `FailTap`, `NoopTap`, `EqualsPredicate`, + `TruthyPredicate`, `RecordHook`, `NestedPipeline`. +- **JSON.** The runner has its own small JSON reader and writer. A runner must + not download anything, so `serde_json` is not used. The core crate does not + get JSON either: principle 5 says that core holds nothing that is not in the + contract. +- **`run.py`.** A `_rust(target)` entry runs + `cargo run --quiet --release --offline --manifest-path conformance/runners/rust/Cargo.toml -- `. + `default_targets()` leaves out `rust:*` when there is no `cargo`, as it + leaves out TypeScript when there is no Node 22.18. The target `rust:run` is + added to `TARGETS`. +- **Benchmark.** `rust:run` is added to `bench.py`. Its number, beside + `python:run` and the compiled Python forms, decides phase 3 (below). +- **CI.** A `rust` job in `ci.yml`: `cargo fmt --check`, `cargo clippy`, + `cargo test`, the example, and `python conformance/run.py rust:run`. +- **Documents.** A Rust column in the "Runtimes" table of API.md, a row in the + README's table of packages, and `rust:run` in MATRIX.md. + +Done when: `rust:run` passes all 27 cases, with nothing unsupported. + +Open questions 3 and 4 of the conformance README (`equals` and `truthy` +across languages) get a Rust answer here. Recommendation: `equals` compares +`Value` by content, and `truthy` follows Python (an empty list or map is +false). No case depends on them yet. + +## A State for each run + +**Done** (2026-10-27). The contract on beta says that a State belongs to one +run, not to the Pipeline (conformance/API.md, "State"). In Rust: + +- **`run(&self, ...)`** in place of `run(&mut self, ...)`, so that a step can + hold the pipeline that it is in (an `Rc`, or a `Weak` to it) and run it + again. The steps were `&self` already. D8 changes with it. +- **`run_with_state(&input, &mut state)`** for the State from the caller; + Rust has no optional argument. `State::new()` makes one. +- **`state()` gives an `Rc`**: the State of the run that started last, + once it has ended. Each run takes a number when it starts, and its State + is kept at the end only if no later run has started. A snapshot cannot + change under a reader, and holding one never blocks a run. The cost: a + value borrowed from it needs the `Rc` in a variable first. +- **Errors in a State are `Arc`** (`SharedError`), so that a State + can be cloned: with the caller's State, the Pipeline keeps a copy for + `state()`. +- **Speed.** Two States take turns, the one that `state()` shows and the one + that the next run fills, so a run still makes no new label String. The + `Rc` of each run costs about 0.03 µs (10 filters: 0.25 to 0.28 µs). + +`rust:run` passes all 30 cases. Breaking the "started last" rule on purpose +fails one case and one unit test. The runner plays `run_again` as the +TypeScript runner does: a function, set for each case, that runs the +pipeline of the case again with a new State. + +## Phase 3: compile + +`codeupipe-compile` for Rust: it reads the source of a named pipeline and +writes one plain Rust module with no use of codeupipe. Same rule as the +others: **code that compiles gives the same output as the pipeline; a shape +that cannot keep that rule is refused, with the place named.** It is a +codeupipe pipeline itself, as the TypeScript compiler is: + +``` +ResolvePipelineFilter the add_steps body to a list of steps (a nested pipeline flattened) +LowerStepsFilter each step to a plain function +EmitModuleFilter the functions to one module +``` + +It depends on `syn` (to read Rust) and `prettyplease` (to write it), as the +TypeScript compiler depends on `typescript`. The core crate stays without +dependencies. + +```bash +codeupipe-compile src/pricing_pipeline.rs:PricingPipeline --out src/compiled/pricing.rs +``` + +Exit codes as in TypeScript: 0 written, 1 refused, 2 could not run. + +### Before it starts: is it worth it? + +In Python the compiled code is 6 to 8 times faster than the dynamic run, +because it removes the interpreter's work. Rust's dynamic run is already +machine code. What a compile removes in Rust is the State, the label strings, +the Hook loop and the call through `dyn Filter`. That could be 2 to 3 times, +or less. **Phase 2 measures `rust:run` first. Phase 3 starts only if the +number leaves room for a gain that is worth a compiler.** If it does not, the +compiler waits, and the Rust row in the README says "no compiler". + +Measured in phase 2 (2026-10-27): it does not, for now. After the work on the +core, a dynamic step costs about 25 ns, and the dynamic run is faster than +compiled Python in four of the five scenarios. The compiler waits until a +real pipeline shows a need. + +### The value type in compiled code + +Plain Python has `dict`, and plain TypeScript has `Map`. Plain Rust has no +JSON-shaped value. So the compiled module must carry one: a small `Value` enum +and its map, written into the module (about 60 lines). It is not an import of +codeupipe, so the rule holds, but it is code that the compiler writes and +that must stay the same as the core's `Value`. A test checks that the two +behave the same. + +### What compiles and what is refused + +The tables of the TypeScript compiler, in Rust: + +| Shape | Plain form | +|---|---| +| A named pipeline (`impl NamedPipeline`) | `pub fn run(input: &Map) -> Result`. `add_steps` holds only `add_*` calls | +| Filter (`call`) | A function on `&mut Map` | +| Valve | `if predicate(&data) { filter(&mut data)?; }` | +| Tap (`observe`) | The call, and its `Err` dropped: a failure does not stop the run | +| Nested pipeline | Its steps, in place | +| A setting of an instance (`self.rate`) | A `const`, from the literal that `DiscountFilter::new(0.1)` gives | +| `get`, `has`, `insert`, `remove`, `merge`, `to_dict` | Operations on the map | + +| Refused | What to do | +|---|---| +| A Hook, or `observe(...)` on the pipeline | Run the pipeline itself | +| An `add_steps` that does more than add steps | Keep only the `add_*` calls | +| An argument that is not a literal | Give literal settings | +| A step that calls its own helper method or another impl | Put the code in the one method | +| `copy`, `with_trace`, `trace_id`, `lineage` | Run the pipeline itself | +| The Payload given to other code | Use it only through its operations | +| A step type from another crate | Compile steps whose source is in the project | + +The fast mode only (no State, no Hooks), as in TypeScript. Python's +observable mode and inline layout come later, if the benchmark asks for them. + +### The `rust:static` target + +The TypeScript runner writes each case as a source file, compiles it and +imports it. Rust cannot load code while it runs, and a `cargo build` for each +case would take minutes. So the static target works in two stages: write +every case's pipeline as source, compile each into one generated crate (one +module for each case), build that crate once, and run it. + +Done when: `rust:static` passes 11 cases and the 16 State and Hook cases as ✅\*, +as `typescript:static` does, and the refusal tests pass. + +## Phase 4: the linter + +Rust's types already check several rules: + +| Rule | In Rust | +|---|---| +| `CUP-I004` a write outside a Filter | A type error (D2) | +| `CUP-A001`, `CUP-A002` a function as a step, `CUP-A008` a function as a Predicate | A type error (D5) | +| `CUP-A009` an object literal as a step | Not possible in Rust | + +The rules that remain need a linter: `CUP-A003` (a step type declared inside +a function), `CUP-A005` (a plain nested `Pipeline`), `CUP-I003` (State read +by a step), `CUP-I005` (one primitive in each file), `CUP-I006` (the file +name), `CUP-I007` (the role suffix), `CUP-I008` (logic outside a primitive), +`CUP-I009` (a marker that allows nothing). `codeupipe-lint` for Rust is a +pipeline on `syn`, as the TypeScript linter is a pipeline on `typescript`. + +This phase is last. The type system covers the rules that matter most for +correctness. The others are about form. + +## Order and size + +| Phase | What | Rough size | +|---|---|---| +| 0 | Settle D1 to D10 | One review of this page | +| 1 | Core crate, example, README | About 600 lines with tests (TypeScript: 415 lines of source) | +| 2 | Runner, `run.py`, bench, CI, documents | About 400 lines, most of it the JSON reader and the `cup_ops` | +| 3 | Compiler and `rust:static`, if the benchmark says so | About 1,000 lines (TypeScript: 851 lines of source) | +| 4 | Linter | About 1,200 lines | + +Phases 1 and 2 are one release: `codeupipe` 0.1.0 for Rust, in the +conformance matrix. Phases 3 and 4 are later versions. diff --git a/examples/pricing/__init__.py b/examples/pricing/__init__.py index b7a9828..9dbfefc 100644 --- a/examples/pricing/__init__.py +++ b/examples/pricing/__init__.py @@ -3,7 +3,7 @@ discount_filter.py Filter takes the Payload, writes to it, returns it mark_bulk_filter.py Filter at_least_predicate.py Predicate a yes-or-no question; the pipeline joins it to a Filter with add_valve - audit_tap.py Tap reads the Payload; its failure never halts the run + audit_tap.py Tap reads the Payload; if it fails, State records it and the run goes on print_steps_hook.py Hook sees each step start pricing_pipeline.py Pipeline where they come together diff --git a/examples/pricing/audit_tap.py b/examples/pricing/audit_tap.py index bc4846f..d195ca0 100644 --- a/examples/pricing/audit_tap.py +++ b/examples/pricing/audit_tap.py @@ -2,8 +2,8 @@ class AuditTap: - """Reads the Payload and returns nothing. This one fails every time, to - show that a Tap failure is recorded and never halts the run.""" + """Reads the Payload and returns nothing. If it fails, State records the + error and the run goes on.""" def observe(self, payload): - raise ConnectionError("audit sink is down") + print("audit:", payload.to_dict()) diff --git a/examples/pricing/pricing_pipeline.py b/examples/pricing/pricing_pipeline.py index 06442db..14b9afe 100644 --- a/examples/pricing/pricing_pipeline.py +++ b/examples/pricing/pricing_pipeline.py @@ -14,5 +14,5 @@ def __init__(self): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) diff --git a/packages/codeupipe-compile-ts/tests/compile.test.ts b/packages/codeupipe-compile-ts/tests/compile.test.ts index 0f73f51..1b57403 100644 --- a/packages/codeupipe-compile-ts/tests/compile.test.ts +++ b/packages/codeupipe-compile-ts/tests/compile.test.ts @@ -31,9 +31,12 @@ test("the compiled pricing example gives the same output as the pipeline", async }); test("the compiled code passes a strict type check", async () => { - const file = join(project({}), "pricing.ts"); + // Every JavaScript host has console, so a Tap may log. Nothing else of a + // browser or of Node is given. + const dir = project({ "host.d.ts": "declare const console: { log(...data: unknown[]): void };\n" }); + const file = join(dir, "pricing.ts"); await compile(PRICING, "PricingPipeline", { outFile: file }); - const program = ts.createProgram([file], { + const program = ts.createProgram([file, join(dir, "host.d.ts")], { strict: true, noEmit: true, target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.NodeNext, moduleResolution: ts.ModuleResolutionKind.NodeNext, lib: ["lib.es2022.d.ts"], types: [], }); diff --git a/packages/codeupipe-rs/Cargo.toml b/packages/codeupipe-rs/Cargo.toml new file mode 100644 index 0000000..cdf432a --- /dev/null +++ b/packages/codeupipe-rs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "codeupipe" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +description = "A Rust library for writing a program as a pipeline of small steps." +license = "Apache-2.0" +repository = "https://github.com/orchestrate-solutions/codeupipe" +readme = "README.md" +keywords = ["pipeline", "filter", "payload", "composition"] +# Not on crates.io yet. A version below 1.0 is not stable. +publish = false + +[[example]] +name = "pricing" +path = "examples/pricing/main.rs" + +[[example]] +name = "typed_pricing" +path = "examples/typed_pricing/main.rs" diff --git a/packages/codeupipe-rs/LICENSE b/packages/codeupipe-rs/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-rs/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-rs/NOTICE b/packages/codeupipe-rs/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-rs/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-rs/README.md b/packages/codeupipe-rs/README.md new file mode 100644 index 0000000..d4bc603 --- /dev/null +++ b/packages/codeupipe-rs/README.md @@ -0,0 +1,222 @@ +# codeupipe for Rust + +codeupipe is a library for writing a program as a pipeline of small steps. +This is the Rust runtime. It has no dependencies. + +It implements the same contract as the Python and TypeScript packages: +[conformance/API.md](../../conformance/API.md). The conformance runner for +Rust is not written yet ([docs/rust-plan.md](../../docs/rust-plan.md), phase 2). + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. + +The crate is not on crates.io yet. Use it from this repository: + +```toml +[dependencies] +codeupipe = { git = "https://github.com/orchestrate-solutions/codeupipe", branch = "beta" } +``` + +## The eight primitives + +| Primitive | What it does | Who writes it | +|---|---|---| +| **Payload** | Carries the data | The library | +| **Filter** | Changes the data | You | +| **Predicate** | Asks a yes-or-no question about the data | You | +| **Valve** | Joins a Predicate to a Filter: the Filter runs when the answer is yes | The library. You add one with `add_valve(predicate, filter)` | +| **Tap** | Reads the data as it goes by | You | +| **Hook** | Is called before and after each step | You | +| **State** | Records which steps ran | The library | +| **Pipeline** | Runs the steps in order | You | + +## Example + +[examples/pricing/](examples/pricing/) has one type in each file. The +pipeline file adds them. + +```rust +// discount_filter.rs +use codeupipe::{Filter, Payload, StepError, Value}; + +pub struct DiscountFilter { + rate: f64, +} + +impl Filter for DiscountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) * (1.0 - self.rate); + payload.insert("price", (price * 100.0).round() / 100.0); + Ok(()) + } +} +``` + +```rust +// at_least_predicate.rs +use codeupipe::{Payload, Predicate, Value}; + +pub struct AtLeastPredicate { + key: String, + minimum: f64, +} + +impl Predicate for AtLeastPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get(&self.key).and_then(Value::as_f64).unwrap_or(0.0) >= self.minimum + } +} +``` + +```rust +// pricing_pipeline.rs +use codeupipe::{NamedPipeline, Pipeline}; + +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} +``` + +```rust +use codeupipe::{Map, Payload, Pipeline}; + +let pipeline = Pipeline::named(PricingPipeline); +let result = pipeline.run(&Payload::from_map(Map::from([("price", 100.0), ("qty", 12.0)])))?; +result.to_dict(); +// {"price": 90, "qty": 12, "bulk": true} +``` + +```bash +cargo run --example pricing +``` + +## A payload of your own + +`Payload` is the default, and it stays the default: a map of JSON-shaped +values that any developer can read, in any runtime of codeupipe. It shows +that anything can go there. + +You can opt in to a type of your own, such as the struct that you would +write anyway. Implement `PipelineData` for it (one line), and the steps of +that pipeline take the struct: + +```rust +#[derive(Clone)] +pub struct Order { + pub price: f64, + pub qty: u32, + pub bulk: bool, +} + +impl PipelineData for Order {} + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + pipeline.add_valve(AtLeastPredicate::new(10), MarkBulkFilter); + pipeline.add_tap(AuditTap); + } +} +``` + +`Filter`, `Predicate`, `Tap`, `Hook`, `NamedPipeline` and `Pipeline` each take +the type as a parameter, which is `Payload` when you leave it out. The rules +stay the same: `run` works on a clone of the input, only a Filter writes, a +Tap failure is recorded and the run goes on, and the labels are the same. A +step for another type is a compile error. Lineage is kept only if the type +implements `PipelineData::stamp`. + +The whole example is in [examples/typed_pricing/](examples/typed_pricing/): + +```bash +cargo run --example typed_pricing +``` + +A typed pipeline has no map to look keys up in, so it is faster: on the +pricing pipeline 0.024 µs a run against 0.18 µs with a Payload, and for 50 +writes 0.46 µs against 4.3 µs (Apple Silicon, release build). + +## Names + +The names are the names of the contract. + +| Contract | Rust | +|---|---| +| `add_filter`, `add_valve`, `add_tap`, `add_pipeline`, `use_hook` | the same | +| `run(p)`, `run(p, state)`, `state`, `labels` | `run(&p)`, `run_with_state(&p, &mut state)`, `state()`, `labels()` | +| `observe(timing, lineage)` | `observe(timing, lineage)`, two `bool`s | +| `get`, `has`, `insert`, `remove`, `merge`, `copy`, `to_dict`, `with_trace` | the same | +| `trace_id`, `lineage` | `trace_id()`, `lineage()` | +| Hook: `before`, `after`, `on_error` | the same | +| A named pipeline: `class PricingPipeline(Pipeline)` | `impl NamedPipeline for PricingPipeline`, then `Pipeline::named(PricingPipeline)` | + +`Filter`, `Predicate`, `Tap` and `Hook` are traits. The methods of `Hook` do +nothing unless you implement them. + +## Differences from Python and TypeScript + +- **A Filter gets `&mut Payload`.** It writes in place, as the Payload is + changed in place in TypeScript, and returns `Ok(())`. A Tap, a Predicate and + a Hook get `&Payload`, so **only a Filter can write, and the compiler checks + it**. A Hook's `on_error` sees the Payload as the failing step left it. +- **What the contract refuses when a step is added is a compile error.** A + closure is not a Filter, and `add_valve(filter, predicate)` does not + type-check. The doc tests in `src/lib.rs` check this. +- **`get` gives `Option<&Value>`.** Absent is `None`. A default is + `.unwrap_or(...)`. `Value` is JSON-shaped, with one number type: `1` and + `1.0` are the same value. +- **`run` is sync, and there is no `run_async` yet.** It takes `&self`, so a + step can hold the pipeline that it is in (an `Rc`) and run it again; each + run writes into a State of its own. Rust has no optional argument, so the + run with the caller's State is `run_with_state(&input, &mut state)`. +- **`state()` gives an `Rc`**: the State of the run that started last, + as a snapshot that a later run does not change. Keep it in a variable to + borrow from it (`let state = pipeline.state();`). Its errors are + `Arc` (`SharedError`), so that a State can be cloned. +- **A failure is an `Err`.** A panic is not a failure: it is not caught, and + a panicking Tap stops the run. +- **`copy()` is deep.** A value shared by two Payloads could not be changed in + place anyway, so the behaviour is the same as the shallow copy of the + contract. +- **The label** is the type name without its path or generic arguments. It + comes from `std::any::type_name`, whose exact text Rust does not promise to + keep the same; the tests check it. +- **A payload of your own** is a type parameter of each primitive (see + above), as in TypeScript. Python takes any object with `copy()`. +- **Timing in WebAssembly.** `std::time::Instant` panics on + `wasm32-unknown-unknown`. Timing is off by default, so a pipeline runs + there as long as `observe(true, ...)` is not called. + +## Development + +The Rust packages are one Cargo workspace. At the root of the repository: + +```bash +cargo test # the tests, and the doc tests +cargo clippy -- -D warnings +cargo fmt --check +cargo run --example pricing +cargo run --example typed_pricing +``` + +One primitive is in each file, and the file name is the type name in +snake_case. + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs b/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs new file mode 100644 index 0000000..4592f4c --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs @@ -0,0 +1,27 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// A Predicate: asks a yes-or-no question and changes nothing. Its settings +/// are on the instance. +pub struct AtLeastPredicate { + key: String, + minimum: f64, +} + +impl AtLeastPredicate { + pub fn new(key: &str, minimum: f64) -> Self { + AtLeastPredicate { + key: key.to_string(), + minimum, + } + } +} + +impl Predicate for AtLeastPredicate { + fn test(&self, payload: &Payload) -> bool { + payload + .get(&self.key) + .and_then(Value::as_f64) + .unwrap_or(0.0) + >= self.minimum + } +} diff --git a/packages/codeupipe-rs/examples/pricing/audit_tap.rs b/packages/codeupipe-rs/examples/pricing/audit_tap.rs new file mode 100644 index 0000000..28d3079 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/audit_tap.rs @@ -0,0 +1,12 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// A Tap: reads the Payload and returns nothing. If it fails, State records +/// the error and the run goes on. +pub struct AuditTap; + +impl Tap for AuditTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + println!("audit: {:?}", payload.to_dict()); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/discount_filter.rs b/packages/codeupipe-rs/examples/pricing/discount_filter.rs new file mode 100644 index 0000000..f1145cd --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/discount_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// A Filter. Its setting is on the instance. +pub struct DiscountFilter { + rate: f64, +} + +impl DiscountFilter { + pub fn new(rate: f64) -> Self { + DiscountFilter { rate } + } +} + +impl Filter for DiscountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) * (1.0 - self.rate); + payload.insert("price", (price * 100.0).round() / 100.0); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/main.rs b/packages/codeupipe-rs/examples/pricing/main.rs new file mode 100644 index 0000000..527c99c --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/main.rs @@ -0,0 +1,35 @@ +//! cargo run --example pricing: run the pipeline with a Hook and show its State. +mod at_least_predicate; +mod audit_tap; +mod discount_filter; +mod mark_bulk_filter; +mod pricing_pipeline; +mod print_steps_hook; + +use codeupipe::{Map, Payload, Pipeline, StepError}; + +use pricing_pipeline::PricingPipeline; +use print_steps_hook::PrintStepsHook; + +fn main() -> Result<(), StepError> { + let mut pipeline = Pipeline::named(PricingPipeline); + pipeline.use_hook(PrintStepsHook); + + println!("steps:"); + let result = pipeline.run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 12.0), + ])))?; + + let state = pipeline.state(); + let errors: Vec<(&str, String)> = state + .errors() + .iter() + .map(|(label, error)| (label.as_str(), error.to_string())) + .collect(); + println!("result: {:?}", result.to_dict()); + println!("executed: {:?}", state.executed()); + println!("skipped: {:?}", state.skipped()); + println!("errors: {errors:?}"); + Ok(()) +} diff --git a/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs b/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs new file mode 100644 index 0000000..3cba368 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs @@ -0,0 +1,11 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// A Filter. +pub struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("bulk", true); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs b/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs new file mode 100644 index 0000000..901db31 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs @@ -0,0 +1,19 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::at_least_predicate::AtLeastPredicate; +use crate::audit_tap::AuditTap; +use crate::discount_filter::DiscountFilter; +use crate::mark_bulk_filter::MarkBulkFilter; + +/// A named pipeline: `Pipeline::named(PricingPipeline)`. +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} diff --git a/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs b/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs new file mode 100644 index 0000000..3132b99 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs @@ -0,0 +1,13 @@ +use codeupipe::{Hook, Payload}; + +/// A Hook: sees the lifecycle. `step` is the label of the step, which is the +/// name of its type. It is `None` for the pipeline itself. +pub struct PrintStepsHook; + +impl Hook for PrintStepsHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + if let Some(step) = step { + println!(" {step}"); + } + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs b/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs new file mode 100644 index 0000000..d53350e --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs @@ -0,0 +1,21 @@ +use codeupipe::Predicate; + +use crate::order::Order; + +/// A Predicate: asks a yes-or-no question and changes nothing. Its setting +/// is on the instance. +pub struct AtLeastPredicate { + minimum: u32, +} + +impl AtLeastPredicate { + pub fn new(minimum: u32) -> Self { + AtLeastPredicate { minimum } + } +} + +impl Predicate for AtLeastPredicate { + fn test(&self, order: &Order) -> bool { + order.qty >= self.minimum + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs b/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs new file mode 100644 index 0000000..602885d --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs @@ -0,0 +1,14 @@ +use codeupipe::{StepError, Tap}; + +use crate::order::Order; + +/// A Tap: reads the Order and returns nothing. If it fails, State records +/// the error and the run goes on. +pub struct AuditTap; + +impl Tap for AuditTap { + fn observe(&self, order: &Order) -> Result<(), StepError> { + println!("audit: {order:?}"); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs b/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs new file mode 100644 index 0000000..c6eafcd --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs @@ -0,0 +1,21 @@ +use codeupipe::{Filter, StepError}; + +use crate::order::Order; + +/// A Filter. Its setting is on the instance. +pub struct DiscountFilter { + rate: f64, +} + +impl DiscountFilter { + pub fn new(rate: f64) -> Self { + DiscountFilter { rate } + } +} + +impl Filter for DiscountFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.price = (order.price * (1.0 - self.rate) * 100.0).round() / 100.0; + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/main.rs b/packages/codeupipe-rs/examples/typed_pricing/main.rs new file mode 100644 index 0000000..1141f9d --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/main.rs @@ -0,0 +1,24 @@ +//! cargo run --example typed_pricing: the pricing example with a struct of +//! your own in place of the Payload. +mod at_least_predicate; +mod audit_tap; +mod discount_filter; +mod mark_bulk_filter; +mod order; +mod pricing_pipeline; + +use codeupipe::{Pipeline, StepError}; + +use order::Order; +use pricing_pipeline::PricingPipeline; + +fn main() -> Result<(), StepError> { + let pipeline = Pipeline::named(PricingPipeline); + let result = pipeline.run(&Order::new(100.0, 12))?; + + let state = pipeline.state(); + println!("result: {result:?}"); + println!("executed: {:?}", state.executed()); + println!("skipped: {:?}", state.skipped()); + Ok(()) +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs b/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs new file mode 100644 index 0000000..bdd8aa3 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, StepError}; + +use crate::order::Order; + +/// A Filter. +pub struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/order.rs b/packages/codeupipe-rs/examples/typed_pricing/order.rs new file mode 100644 index 0000000..a24fe35 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/order.rs @@ -0,0 +1,23 @@ +use codeupipe::PipelineData; + +/// The data of this pipeline: a struct in place of the Payload. Each step +/// below takes an `Order`, and the compiler checks every field it uses. +#[derive(Clone, Debug, PartialEq)] +pub struct Order { + pub price: f64, + pub qty: u32, + pub bulk: bool, +} + +impl Order { + pub fn new(price: f64, qty: u32) -> Self { + Order { + price, + qty, + bulk: false, + } + } +} + +/// The one line that lets a Pipeline carry an Order. +impl PipelineData for Order {} diff --git a/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs b/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs new file mode 100644 index 0000000..95ab703 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs @@ -0,0 +1,20 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::at_least_predicate::AtLeastPredicate; +use crate::audit_tap::AuditTap; +use crate::discount_filter::DiscountFilter; +use crate::mark_bulk_filter::MarkBulkFilter; +use crate::order::Order; + +/// The pricing pipeline of `examples/pricing`, carrying an Order. +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new(10), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} diff --git a/packages/codeupipe-rs/src/filter.rs b/packages/codeupipe-rs/src/filter.rs new file mode 100644 index 0000000..384fc15 --- /dev/null +++ b/packages/codeupipe-rs/src/filter.rs @@ -0,0 +1,10 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Filter: changes the data. A failure is an `Err`, and it stops the run. +/// +/// The Filter gets the Payload itself and writes to it in place. It is the +/// only step that can write: the others get `&Payload`. +pub trait Filter

{ + fn call(&self, payload: &mut P) -> Result<(), StepError>; +} diff --git a/packages/codeupipe-rs/src/hook.rs b/packages/codeupipe-rs/src/hook.rs new file mode 100644 index 0000000..43776a1 --- /dev/null +++ b/packages/codeupipe-rs/src/hook.rs @@ -0,0 +1,18 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Hook: sees the lifecycle of a run. Implement the methods that you need; +/// the others do nothing. +/// +/// `step` is the label of the step, or `None` for the pipeline itself. A Hook +/// reads the Payload and does not write to it. +pub trait Hook

{ + /// Before a step, or before the pipeline starts (`step` is `None`). + fn before(&self, _step: Option<&str>, _payload: &P) {} + + /// After a step, or after the pipeline ends (`step` is `None`). + fn after(&self, _step: Option<&str>, _payload: &P) {} + + /// When a Filter fails: for the step, then for the pipeline (`step` is `None`). + fn on_error(&self, _step: Option<&str>, _error: &StepError, _payload: &P) {} +} diff --git a/packages/codeupipe-rs/src/lib.rs b/packages/codeupipe-rs/src/lib.rs new file mode 100644 index 0000000..b7a3f5d --- /dev/null +++ b/packages/codeupipe-rs/src/lib.rs @@ -0,0 +1,112 @@ +//! codeupipe: the eight primitives. +//! +//! Payload, Filter, Predicate, Valve, Tap, Hook, State and Pipeline, as the +//! contract (conformance/API.md) defines them. A Valve is added with +//! `pipeline.add_valve(predicate, filter)`; its type is not a public name. +//! +//! ``` +//! use codeupipe::{Filter, Map, Payload, Pipeline, StepError, Value}; +//! +//! struct GreetFilter; +//! +//! impl Filter for GreetFilter { +//! fn call(&self, payload: &mut Payload) -> Result<(), StepError> { +//! let name = payload.get("name").and_then(Value::as_str).unwrap_or("World").to_string(); +//! payload.insert("greeting", format!("Hello, {name}!")); +//! Ok(()) +//! } +//! } +//! +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_filter(GreetFilter); +//! let out = pipeline.run(&Payload::from_map(Map::from([("name", "Ada")])))?; +//! assert_eq!(out.get("greeting"), Some(&Value::from("Hello, Ada!"))); +//! assert_eq!(pipeline.state().executed(), ["GreetFilter"]); +//! # Ok::<(), StepError>(()) +//! ``` +//! +//! # What the type system refuses +//! +//! The contract refuses a step without the method of its role when it is +//! added. In Rust that is a compile error. +//! +//! A function is not a step: +//! +//! ```compile_fail +//! use codeupipe::{Payload, Pipeline, StepError}; +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_filter(|_payload: &mut Payload| -> Result<(), StepError> { Ok(()) }); +//! ``` +//! +//! The condition of a Valve comes first: +//! +//! ```compile_fail +//! use codeupipe::{Filter, Payload, Pipeline, Predicate, StepError}; +//! struct OnPredicate; +//! impl Predicate for OnPredicate { +//! fn test(&self, _payload: &Payload) -> bool { true } +//! } +//! struct SetFilter; +//! impl Filter for SetFilter { +//! fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { Ok(()) } +//! } +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_valve(SetFilter, OnPredicate); +//! ``` +//! +//! A pipeline carries one type. A Filter for a type of your own +//! (`PipelineData`) does not go into a pipeline of `Payload`: +//! +//! ```compile_fail +//! use codeupipe::{Filter, Payload, Pipeline, PipelineData, StepError}; +//! #[derive(Clone)] +//! struct Order { price: f64 } +//! impl PipelineData for Order {} +//! struct DiscountFilter; +//! impl Filter for DiscountFilter { +//! fn call(&self, order: &mut Order) -> Result<(), StepError> { order.price *= 0.9; Ok(()) } +//! } +//! let mut pipeline: Pipeline = Pipeline::new(); +//! pipeline.add_filter(DiscountFilter); +//! ``` +//! +//! Only a Filter writes. A Tap gets `&Payload`: +//! +//! ```compile_fail +//! use codeupipe::{Payload, StepError, Tap}; +//! struct WritingTap; +//! impl Tap for WritingTap { +//! fn observe(&self, payload: &Payload) -> Result<(), StepError> { +//! payload.insert("seen", true); +//! Ok(()) +//! } +//! } +//! ``` + +mod filter; +mod hook; +mod map; +mod named_pipeline; +mod payload; +mod pipeline; +mod pipeline_data; +mod predicate; +mod state; +mod step_error; +mod step_label; +mod tap; +mod value; +mod valve; + +pub use filter::Filter; +pub use hook::Hook; +pub use map::Map; +pub use named_pipeline::NamedPipeline; +pub use payload::Payload; +pub use pipeline::Pipeline; +pub use pipeline_data::PipelineData; +pub use predicate::Predicate; +pub use state::{SharedError, State}; +pub use step_error::StepError; +pub use tap::Tap; +pub use value::Value; diff --git a/packages/codeupipe-rs/src/map.rs b/packages/codeupipe-rs/src/map.rs new file mode 100644 index 0000000..954a1ed --- /dev/null +++ b/packages/codeupipe-rs/src/map.rs @@ -0,0 +1,130 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use crate::value::Value; + +/// A map with more keys than this has an index; a smaller one is searched in +/// order, which is faster than hashing for a few keys. +const INDEX_FROM: usize = 8; + +/// A map from text keys to values. Keys keep the order in which they were +/// first inserted, as a Python dict and a JavaScript Map do. +/// +/// Two maps are equal when they have the same entries, in any order, as two +/// JSON objects are. +#[derive(Clone, Default)] +pub struct Map { + // A key is made once and shared, by the index and by a clone of the map. + entries: Vec<(Arc, Value)>, + index: Option, usize>>, +} + +impl Map { + pub fn new() -> Self { + Map::default() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn position(&self, key: &str) -> Option { + match &self.index { + Some(index) => index.get(key).copied(), + None => self.entries.iter().position(|(known, _)| &**known == key), + } + } + + pub fn contains_key(&self, key: &str) -> bool { + self.position(key).is_some() + } + + pub fn get(&self, key: &str) -> Option<&Value> { + self.position(key).map(|at| &self.entries[at].1) + } + + /// Set `key` to `value`. A key that is there keeps its place. + pub fn insert(&mut self, key: impl AsRef, value: impl Into) { + let value = value.into(); + let key = key.as_ref(); + if let Some(at) = self.position(key) { + self.entries[at].1 = value; + return; + } + let key: Arc = Arc::from(key); + let at = self.entries.len(); + self.entries.push((Arc::clone(&key), value)); + match &mut self.index { + Some(index) => { + index.insert(key, at); + } + None if self.entries.len() > INDEX_FROM => { + // Room for a few times the size, so that a map that is filled + // key by key does not grow its index at every doubling. + let mut index = HashMap::with_capacity(4 * INDEX_FROM); + for (at, (key, _)) in self.entries.iter().enumerate() { + index.insert(Arc::clone(key), at); + } + self.index = Some(index); + } + None => {} + } + } + + /// Remove `key`, and give its value. Nothing happens if it is absent. + pub fn remove(&mut self, key: &str) -> Option { + let at = self.position(key)?; + let (_, value) = self.entries.remove(at); + if let Some(index) = &mut self.index { + index.remove(key); + for (moved, (key, _)) in self.entries.iter().enumerate().skip(at) { + index.insert(Arc::clone(key), moved); + } + } + Some(value) + } + + pub fn keys(&self) -> impl Iterator { + self.entries.iter().map(|(key, _)| &**key) + } + + pub fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(key, value)| (&**key, value)) + } +} + +impl PartialEq for Map { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() + && self + .iter() + .all(|(key, value)| other.get(key) == Some(value)) + } +} + +impl fmt::Debug for Map { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries(self.iter()).finish() + } +} + +impl, V: Into> FromIterator<(K, V)> for Map { + fn from_iter>(entries: I) -> Self { + let mut map = Map::new(); + for (key, value) in entries { + map.insert(key, value); + } + map + } +} + +impl, V: Into, const N: usize> From<[(K, V); N]> for Map { + fn from(entries: [(K, V); N]) -> Self { + entries.into_iter().collect() + } +} diff --git a/packages/codeupipe-rs/src/named_pipeline.rs b/packages/codeupipe-rs/src/named_pipeline.rs new file mode 100644 index 0000000..5e93539 --- /dev/null +++ b/packages/codeupipe-rs/src/named_pipeline.rs @@ -0,0 +1,12 @@ +use crate::payload::Payload; +use crate::pipeline::Pipeline; + +/// A pipeline with a name: a type that adds the steps. +/// +/// `Pipeline::named(UserOnboarding)` makes the pipeline, and its label is +/// `UserOnboarding`. Rust has no class to extend, so this trait is the named +/// type of the contract (`class UserOnboarding(Pipeline)` in Python). +/// `add_steps` only adds steps. +pub trait NamedPipeline

{ + fn add_steps(&self, pipeline: &mut Pipeline

); +} diff --git a/packages/codeupipe-rs/src/payload.rs b/packages/codeupipe-rs/src/payload.rs new file mode 100644 index 0000000..e44aee0 --- /dev/null +++ b/packages/codeupipe-rs/src/payload.rs @@ -0,0 +1,102 @@ +use crate::map::Map; +use crate::pipeline_data::PipelineData; +use crate::value::Value; + +/// Payload: the data that moves through a pipeline. +/// +/// One keyed container. Every write changes it in place and returns it, so +/// writes can be chained. `copy()` gives a separate Payload when you want to +/// keep the one you have. +#[derive(Clone, Debug, Default)] +pub struct Payload { + data: Map, + trace_id: Option, + lineage: Vec, +} + +impl Payload { + /// An empty Payload. + pub fn new() -> Self { + Payload::default() + } + + /// A Payload with the contents of `data`. The map is moved in, so the + /// caller's map cannot be changed through the Payload. + pub fn from_map(data: Map) -> Self { + Payload { + data, + ..Payload::default() + } + } + + /// The value for `key`, or absent. A default is `.unwrap_or(...)`. + pub fn get(&self, key: &str) -> Option<&Value> { + self.data.get(key) + } + + /// Whether `key` is present (also when its value is null). + pub fn has(&self, key: &str) -> bool { + self.data.contains_key(key) + } + + /// Set `key` to `value` in this Payload, and return it. + pub fn insert + Into>( + &mut self, + key: K, + value: impl Into, + ) -> &mut Self { + self.data.insert(key, value); + self + } + + /// Remove `key` from this Payload (nothing happens if it is absent), and return it. + pub fn remove(&mut self, key: &str) -> &mut Self { + self.data.remove(key); + self + } + + /// Write every key of `other` into this Payload (`other` wins), and return it. + pub fn merge(&mut self, other: &Payload) -> &mut Self { + for (key, value) in other.data.iter() { + self.data.insert(key, value.clone()); + } + if self.trace_id.is_none() { + self.trace_id = other.trace_id.clone(); + } + self.lineage.extend(other.lineage.iter().cloned()); + self + } + + /// A separate Payload with the same contents, trace id and lineage. + pub fn copy(&self) -> Payload { + self.clone() + } + + /// The contents as a new map. + pub fn to_dict(&self) -> Map { + self.data.clone() + } + + /// Set the trace id on this Payload, and return it. + pub fn with_trace(&mut self, trace_id: impl Into) -> &mut Self { + self.trace_id = Some(trace_id.into()); + self + } + + /// The trace id, for following one Payload across systems. + pub fn trace_id(&self) -> Option<&str> { + self.trace_id.as_deref() + } + + /// The steps this Payload has passed through, when lineage is on. + pub fn lineage(&self) -> &[String] { + &self.lineage + } +} + +impl PipelineData for Payload { + /// Lineage: the label of each step, in `lineage()`. + fn stamp(&mut self, label: &str) { + self.lineage.push(label.to_string()); + } +} diff --git a/packages/codeupipe-rs/src/pipeline.rs b/packages/codeupipe-rs/src/pipeline.rs new file mode 100644 index 0000000..339e6fb --- /dev/null +++ b/packages/codeupipe-rs/src/pipeline.rs @@ -0,0 +1,323 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::fmt; +use std::rc::Rc; +use std::time::Instant; + +use crate::filter::Filter; +use crate::hook::Hook; +use crate::named_pipeline::NamedPipeline; +use crate::payload::Payload; +use crate::pipeline_data::PipelineData; +use crate::predicate::Predicate; +use crate::state::State; +use crate::step_error::StepError; +use crate::step_label::label_of; +use crate::tap::Tap; +use crate::valve::Valve; + +/// A step that Hooks see: it runs, and its failure stops the run. +enum Gated

{ + Filter(Box>), + Valve(Valve

), + Pipeline(Box>), +} + +enum Step

{ + Tap(Box>), + Gated(Gated

), +} + +struct Labeled

{ + label: String, + step: Step

, +} + +/// Pipeline: runs the steps in order. +/// +/// Filters change the Payload, Valves gate a Filter, Taps read, Hooks see the +/// lifecycle, and State records what ran. Add steps with `add_filter`, +/// `add_valve`, `add_tap` and `add_pipeline`, add Hooks with `use_hook`, and +/// run with `pipeline.run(&payload)`. +/// +/// It carries a `Payload` unless you give it a type of your own +/// (`Pipeline`, see `PipelineData`). +pub struct Pipeline

{ + label: String, + steps: Vec>, + uses: HashMap, + hooks: Vec>>, + runs: Runs, + timing: bool, + lineage: bool, +} + +/// The runs of a pipeline. Each run writes into a State of its own; this +/// keeps the State of the run that started last, for `state()`. +#[derive(Default)] +struct Runs { + /// The number of runs that have started; a run's number is its ticket. + started: Cell, + /// The State of the run that started last, and that has ended. + last: RefCell>, + /// A State that no run and no reader has, kept for its storage. + spare: RefCell>, +} + +impl Runs { + fn start(&self) -> u64 { + let ticket = self.started.get() + 1; + self.started.set(ticket); + ticket + } + + /// A State for a run, with the storage of an earlier one when there is one. + fn new_state(&self) -> State { + let mut state = self.spare.take().unwrap_or_default(); + state.reset(); + state + } + + /// Keep `state` as the State of the pipeline, unless a run that started + /// later has ended already (a run inside this run). + fn end(&self, ticket: u64, state: State) { + if ticket != self.started.get() { + self.spare.replace(Some(state)); + return; + } + let earlier = self.last.replace(Rc::new(state)); + if let Ok(earlier) = Rc::try_unwrap(earlier) { + self.spare.replace(Some(earlier)); + } + } +} + +impl Default for Pipeline

{ + fn default() -> Self { + Pipeline::new() + } +} + +impl

fmt::Debug for Pipeline

{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pipeline") + .field("label", &self.label) + .field("steps", &self.labels()) + .finish() + } +} + +impl

Pipeline

{ + /// The label of each step, in order. + pub fn labels(&self) -> Vec { + self.steps.iter().map(|step| step.label.clone()).collect() + } +} + +impl Pipeline

{ + /// A pipeline with no name. Nested, its label is `Pipeline`. + pub fn new() -> Self { + Pipeline { + label: "Pipeline".to_string(), + steps: Vec::new(), + uses: HashMap::new(), + hooks: Vec::new(), + runs: Runs::default(), + timing: false, + lineage: false, + } + } + + /// The pipeline that `named` adds the steps of. Its label is the name of + /// the type of `named`. + pub fn named>(named: N) -> Self { + let mut pipeline = Pipeline::new(); + pipeline.label = label_of::(); + named.add_steps(&mut pipeline); + pipeline + } + + /// The State of the run that started last, once it has ended. + /// + /// A State belongs to one run, and this is for a caller with one run at a + /// time. It is a snapshot: a later run does not change it. A caller with + /// more than one run at a time gives each run its own State + /// (`run_with_state`). + pub fn state(&self) -> Rc { + Rc::clone(&self.runs.last.borrow()) + } + + /// The type name of the step, with a number from its second use in this pipeline. + fn next_label(&mut self, base: String) -> String { + let count = self.uses.entry(base.clone()).or_insert(0); + *count += 1; + if *count == 1 { + base + } else { + format!("{base}#{count}") + } + } + + fn push(&mut self, base: String, step: Step

) { + let label = self.next_label(base); + self.steps.push(Labeled { label, step }); + } + + /// Add a Filter. + pub fn add_filter + 'static>(&mut self, filter: F) { + self.push( + label_of::(), + Step::Gated(Gated::Filter(Box::new(filter))), + ); + } + + /// Add a Valve: `filter` runs only when `predicate.test(payload)` is true. + /// + /// The condition comes first, then the Filter that it controls. When the + /// test is false the Payload passes through unchanged and the step is + /// recorded as skipped. The step has the label of its Filter. + pub fn add_valve + 'static, F: Filter

+ 'static>( + &mut self, + predicate: Q, + filter: F, + ) { + let valve = Valve { + predicate: Box::new(predicate), + filter: Box::new(filter), + }; + self.push(label_of::(), Step::Gated(Gated::Valve(valve))); + } + + /// Add a Tap. + pub fn add_tap + 'static>(&mut self, tap: T) { + self.push(label_of::(), Step::Tap(Box::new(tap))); + } + + /// Nest a Pipeline as one step of this Pipeline. + pub fn add_pipeline(&mut self, pipeline: Pipeline

) { + self.push( + pipeline.label.clone(), + Step::Gated(Gated::Pipeline(Box::new(pipeline))), + ); + } + + /// Add a Hook. + pub fn use_hook + 'static>(&mut self, hook: H) { + self.hooks.push(Box::new(hook)); + } + + /// Turn timing (the time of each step, in State) and lineage (labels on + /// the Payload) on or off. Both are off by default. + pub fn observe(&mut self, timing: bool, lineage: bool) { + self.timing = timing; + self.lineage = lineage; + } + + /// Run the steps in order on a copy of `input`, and return the result. + /// + /// A Filter failure stops the run and is returned after the Hooks have + /// seen it. A Tap failure is recorded in State and the run continues. + /// The run writes into a new State, which `state()` gives afterwards. + /// + /// It takes `&self`: a step can hold the pipeline that it is in (an `Rc`) + /// and run it again, and each run has its own State. + pub fn run(&self, input: &P) -> Result { + let ticket = self.runs.start(); + let mut state = self.runs.new_state(); + let mut payload = input.clone(); // the caller's value is not changed + let result = self.run_steps(&mut payload, &mut state); + self.runs.end(ticket, state); + result.map(|()| payload) + } + + /// `run`, writing into `state`, which the caller gives. A caller with more + /// than one run at a time gives each run its own State. + pub fn run_with_state(&self, input: &P, state: &mut State) -> Result { + let ticket = self.runs.start(); + let mut payload = input.clone(); // the caller's value is not changed + let result = self.run_steps(&mut payload, state); + self.runs.end(ticket, state.clone()); + result.map(|()| payload) + } + + fn run_steps(&self, payload: &mut P, state: &mut State) -> Result<(), StepError> { + let Pipeline { + steps, + hooks, + timing, + lineage, + .. + } = self; + + for hook in hooks.iter() { + hook.before(None, payload); + } + + for Labeled { label, step } in steps.iter() { + let gated = match step { + Step::Tap(tap) => { + if let Err(error) = tap.observe(payload) { + state.record_error(label, error); + } + state.mark_executed(label); + continue; + } + Step::Gated(gated) => gated, + }; + + let started = if *timing { Some(Instant::now()) } else { None }; + for hook in hooks.iter() { + hook.before(Some(label), payload); + } + + match run_gated(gated, payload) { + Ok(true) => state.mark_executed(label), + Ok(false) => state.mark_skipped(label), + Err(error) => { + if let Some(started) = started { + state.record_timing(label, started.elapsed()); + } + // The failing step first, then the pipeline: on_error + // fires at both levels, as before and after do. + for hook in hooks.iter() { + hook.on_error(Some(label), &error, payload); + } + for hook in hooks.iter() { + hook.on_error(None, &error, payload); + } + return Err(error); + } + } + + for hook in hooks.iter() { + hook.after(Some(label), payload); + } + if let Some(started) = started { + state.record_timing(label, started.elapsed()); + } + if *lineage { + payload.stamp(label); + } + } + + for hook in hooks.iter() { + hook.after(None, payload); + } + Ok(()) + } +} + +/// Run one step that Hooks see. `Ok(false)` is a Valve that skipped its Filter. +fn run_gated(gated: &Gated

, payload: &mut P) -> Result { + match gated { + Gated::Filter(filter) => filter.call(payload).map(|()| true), + Gated::Valve(valve) if !valve.predicate.test(payload) => Ok(false), + Gated::Valve(valve) => valve.filter.call(payload).map(|()| true), + Gated::Pipeline(inner) => { + // A run of its own, with its own State: a failure inside leaves this + // Payload as it was. + *payload = inner.run(payload)?; + Ok(true) + } + } +} diff --git a/packages/codeupipe-rs/src/pipeline_data.rs b/packages/codeupipe-rs/src/pipeline_data.rs new file mode 100644 index 0000000..75ac028 --- /dev/null +++ b/packages/codeupipe-rs/src/pipeline_data.rs @@ -0,0 +1,39 @@ +/// What a Pipeline carries from step to step. +/// +/// `Payload` is the default: a map of JSON-shaped values that any developer +/// can read, and that every runtime of codeupipe has. A type of your own can +/// take its place, such as a struct with the fields that your steps use: +/// +/// ``` +/// use codeupipe::{Filter, Pipeline, PipelineData, StepError}; +/// +/// #[derive(Clone)] +/// struct Order { price: f64 } +/// +/// impl PipelineData for Order {} +/// +/// struct DiscountFilter; +/// +/// impl Filter for DiscountFilter { +/// fn call(&self, order: &mut Order) -> Result<(), StepError> { +/// order.price *= 0.9; +/// Ok(()) +/// } +/// } +/// +/// let mut pipeline = Pipeline::new(); +/// pipeline.add_filter(DiscountFilter); +/// assert_eq!(pipeline.run(&Order { price: 100.0 })?.price, 90.0); +/// # Ok::<(), StepError>(()) +/// ``` +/// +/// The Filters, Predicates, Taps and Hooks of that pipeline then take the +/// type, and a step for another type is a compile error. The rules stay the +/// same: `run` works on a clone of the input, only a Filter writes, a Tap +/// failure is recorded and the run goes on. +pub trait PipelineData: Clone + 'static { + /// Called after each step when lineage is on (`observe(_, true)`), with + /// the label of the step. `Payload` keeps the labels; a type of your own + /// keeps them only if it implements this. + fn stamp(&mut self, _label: &str) {} +} diff --git a/packages/codeupipe-rs/src/predicate.rs b/packages/codeupipe-rs/src/predicate.rs new file mode 100644 index 0000000..4a5c43f --- /dev/null +++ b/packages/codeupipe-rs/src/predicate.rs @@ -0,0 +1,9 @@ +use crate::payload::Payload; + +/// Predicate: asks a yes-or-no question about the data. +/// +/// A Predicate is a type. Its settings (a threshold, a flag) are on the +/// instance. It reads the Payload and does not write to it. +pub trait Predicate

{ + fn test(&self, payload: &P) -> bool; +} diff --git a/packages/codeupipe-rs/src/state.rs b/packages/codeupipe-rs/src/state.rs new file mode 100644 index 0000000..d9efd87 --- /dev/null +++ b/packages/codeupipe-rs/src/state.rs @@ -0,0 +1,113 @@ +use std::error::Error; +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use crate::step_error::StepError; + +/// An error that State keeps. It is shared, so that a State can be cloned. +pub type SharedError = Arc; + +/// State: what happened in one run. +/// +/// Which steps executed, which a Valve skipped, the errors, and the timings +/// when timing is on. A State belongs to one run: give one to +/// `pipeline.run_with_state`, or read the State of the run that started last +/// with `pipeline.state()`. +#[derive(Clone, Debug, Default)] +pub struct State { + executed: Labels, + skipped: Labels, + errors: Vec<(String, SharedError)>, + timings: Vec<(String, Duration)>, +} + +/// A list of labels whose Strings are kept from one run to the next and +/// written over, so that a run like the last one makes no new String. +#[derive(Clone, Default)] +struct Labels { + all: Vec, + len: usize, +} + +impl Labels { + fn push(&mut self, label: &str) { + match self.all.get_mut(self.len) { + Some(slot) => { + slot.clear(); + slot.push_str(label); + } + None => self.all.push(label.to_string()), + } + self.len += 1; + } + + fn as_slice(&self) -> &[String] { + &self.all[..self.len] + } + + fn clear(&mut self) { + self.len = 0; + } +} + +impl fmt::Debug for Labels { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.as_slice()).finish() + } +} + +impl State { + /// An empty State, for a caller that gives the State of a run + /// (`pipeline.run_with_state(&input, &mut state)`). + pub fn new() -> Self { + State::default() + } + + /// Step labels, in order. + pub fn executed(&self) -> &[String] { + self.executed.as_slice() + } + + /// Steps that a Valve did not run. + pub fn skipped(&self) -> &[String] { + self.skipped.as_slice() + } + + /// Pairs of label and error. + pub fn errors(&self) -> &[(String, SharedError)] { + &self.errors + } + + /// Label and time, in order, when timing is on. + pub fn timings(&self) -> &[(String, Duration)] { + &self.timings + } + + /// Empty, for a new run. The storage is kept. + pub(crate) fn reset(&mut self) { + self.executed.clear(); + self.skipped.clear(); + self.errors.clear(); + self.timings.clear(); + } + + pub(crate) fn mark_executed(&mut self, label: &str) { + self.executed.push(label); + } + + pub(crate) fn mark_skipped(&mut self, label: &str) { + self.skipped.push(label); + } + + pub(crate) fn record_error(&mut self, label: &str, error: StepError) { + self.errors.push((label.to_string(), Arc::from(error))); + } + + pub(crate) fn record_timing(&mut self, label: &str, time: Duration) { + match self.timings.iter_mut().find(|(known, _)| known == label) { + Some(entry) => entry.1 = time, + None => self.timings.push((label.to_string(), time)), + } + } +} diff --git a/packages/codeupipe-rs/src/step_error.rs b/packages/codeupipe-rs/src/step_error.rs new file mode 100644 index 0000000..a36d19a --- /dev/null +++ b/packages/codeupipe-rs/src/step_error.rs @@ -0,0 +1,3 @@ +/// The error of a step that fails. Any error type converts into it, and so +/// does text: `Err("sink is down".into())`. +pub type StepError = Box; diff --git a/packages/codeupipe-rs/src/step_label.rs b/packages/codeupipe-rs/src/step_label.rs new file mode 100644 index 0000000..030c3d7 --- /dev/null +++ b/packages/codeupipe-rs/src/step_label.rs @@ -0,0 +1,7 @@ +/// The label of a step: the name of its type, without its path and without +/// generic arguments. `my_app::pricing::DiscountFilter` gives `DiscountFilter`. +pub(crate) fn label_of() -> String { + let full = std::any::type_name::(); + let path = full.split('<').next().unwrap_or(full); + path.rsplit("::").next().unwrap_or(path).to_string() +} diff --git a/packages/codeupipe-rs/src/tap.rs b/packages/codeupipe-rs/src/tap.rs new file mode 100644 index 0000000..3a4f9f5 --- /dev/null +++ b/packages/codeupipe-rs/src/tap.rs @@ -0,0 +1,8 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Tap: reads the data as it goes by. It does not write to the Payload. +/// A Tap failure is recorded in State and never stops the run. +pub trait Tap

{ + fn observe(&self, payload: &P) -> Result<(), StepError>; +} diff --git a/packages/codeupipe-rs/src/value.rs b/packages/codeupipe-rs/src/value.rs new file mode 100644 index 0000000..87a20c6 --- /dev/null +++ b/packages/codeupipe-rs/src/value.rs @@ -0,0 +1,123 @@ +use std::fmt; + +use crate::map::Map; + +/// A JSON-shaped value: what a Payload holds under a key. +/// +/// There is one number type, as in JSON and JavaScript: `1` and `1.0` are +/// the same value. `true` is never `1`. +#[derive(Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Number(f64), + String(String), + List(Vec), + Map(Map), +} + +impl Value { + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(value) => Some(*value), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Value::Number(value) => Some(*value), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(value) => Some(value), + _ => None, + } + } + + pub fn as_list(&self) -> Option<&[Value]> { + match self { + Value::List(value) => Some(value), + _ => None, + } + } + + pub fn as_map(&self) -> Option<&Map> { + match self { + Value::Map(value) => Some(value), + _ => None, + } + } +} + +/// Written as JSON is, so that a printed Payload reads like the data. +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Value::Null => f.write_str("null"), + Value::Bool(value) => write!(f, "{value}"), + Value::Number(value) if value.fract() == 0.0 && value.abs() < 1e15 => { + write!(f, "{}", *value as i64) + } + Value::Number(value) => write!(f, "{value}"), + Value::String(value) => write!(f, "{value:?}"), + Value::List(values) => f.debug_list().entries(values).finish(), + Value::Map(map) => write!(f, "{map:?}"), + } + } +} + +impl From for Value { + fn from(value: bool) -> Self { + Value::Bool(value) + } +} + +macro_rules! number_from { + ($($t:ty),*) => { + $(impl From<$t> for Value { + fn from(value: $t) -> Self { + Value::Number(value as f64) + } + })* + }; +} + +number_from!(f64, f32, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + +impl From<&str> for Value { + fn from(value: &str) -> Self { + Value::String(value.to_string()) + } +} + +impl From for Value { + fn from(value: String) -> Self { + Value::String(value) + } +} + +impl From> for Value { + fn from(value: Vec) -> Self { + Value::List(value) + } +} + +impl From for Value { + fn from(value: Map) -> Self { + Value::Map(value) + } +} + +impl> From> for Value { + fn from(value: Option) -> Self { + value.map_or(Value::Null, Into::into) + } +} diff --git a/packages/codeupipe-rs/src/valve.rs b/packages/codeupipe-rs/src/valve.rs new file mode 100644 index 0000000..4bc327e --- /dev/null +++ b/packages/codeupipe-rs/src/valve.rs @@ -0,0 +1,11 @@ +use crate::filter::Filter; +use crate::predicate::Predicate; + +/// Valve: joins a Predicate to a Filter. The Filter runs only when the +/// Predicate is true. +/// +/// `Pipeline::add_valve(predicate, filter)` makes one. It is not a public name. +pub(crate) struct Valve

{ + pub(crate) predicate: Box>, + pub(crate) filter: Box>, +} diff --git a/packages/codeupipe-rs/tests/map.rs b/packages/codeupipe-rs/tests/map.rs new file mode 100644 index 0000000..a57eb87 --- /dev/null +++ b/packages/codeupipe-rs/tests/map.rs @@ -0,0 +1,90 @@ +use codeupipe::{Map, Value}; + +/// A map with `n` keys k0, k1, ... in order, each with its number. +fn numbered(n: usize) -> Map { + (0..n).map(|i| (format!("k{i}"), i)).collect() +} + +// A small map is searched in order and a large one through an index, so each +// test runs on both sides of the size where the index starts. +const SIZES: [usize; 4] = [3, 8, 9, 40]; + +#[test] +fn get_finds_every_key() { + for size in SIZES { + let map = numbered(size); + for i in 0..size { + assert_eq!(map.get(&format!("k{i}")), Some(&Value::from(i)), "{size}"); + } + assert_eq!(map.get("absent"), None); + assert!(!map.contains_key("absent")); + } +} + +#[test] +fn an_insert_of_a_key_that_is_there_keeps_its_place() { + for size in SIZES { + let mut map = numbered(size); + map.insert("k1", "new"); + assert_eq!(map.len(), size); + assert_eq!(map.keys().nth(1), Some("k1")); + assert_eq!(map.get("k1"), Some(&Value::from("new"))); + } +} + +#[test] +fn a_remove_keeps_the_order_and_every_other_key() { + for size in SIZES { + let mut map = numbered(size); + assert_eq!(map.remove("k1"), Some(Value::from(1))); + assert_eq!(map.remove("k1"), None); + assert_eq!(map.len(), size - 1); + let want: Vec = (0..size) + .filter(|&i| i != 1) + .map(|i| format!("k{i}")) + .collect(); + assert_eq!(map.keys().collect::>(), want); + for i in (0..size).filter(|&i| i != 1) { + assert_eq!(map.get(&format!("k{i}")), Some(&Value::from(i)), "{size}"); + } + map.insert("k1", "back"); + assert_eq!(map.keys().last(), Some("k1")); + assert_eq!(map.get("k1"), Some(&Value::from("back"))); + } +} + +#[test] +fn a_map_that_grows_past_the_index_and_shrinks_back_still_finds_its_keys() { + let mut map = numbered(20); + for i in 0..18 { + map.remove(&format!("k{i}")); + } + assert_eq!(map.keys().collect::>(), ["k18", "k19"]); + assert_eq!(map.get("k19"), Some(&Value::from(19))); + map.insert("k0", 0); + assert_eq!(map.get("k0"), Some(&Value::from(0))); +} + +#[test] +fn a_clone_is_separate() { + for size in SIZES { + let map = numbered(size); + let mut other = map.clone(); + other.insert("k0", "changed"); + other.insert("extra", true); + assert_eq!(map.get("k0"), Some(&Value::from(0))); + assert!(!map.contains_key("extra")); + } +} + +#[test] +fn two_maps_with_the_same_entries_in_another_order_are_equal() { + for size in SIZES { + let forward = numbered(size); + let backward: Map = (0..size).rev().map(|i| (format!("k{i}"), i)).collect(); + assert_eq!(forward, backward); + let mut different = numbered(size); + different.insert("k0", "x"); + assert_ne!(forward, different); + } +} diff --git a/packages/codeupipe-rs/tests/payload.rs b/packages/codeupipe-rs/tests/payload.rs new file mode 100644 index 0000000..4e90e4b --- /dev/null +++ b/packages/codeupipe-rs/tests/payload.rs @@ -0,0 +1,168 @@ +use codeupipe::{Map, Payload, Value}; + +#[test] +fn the_map_that_the_caller_gives_is_copied() { + let data = Map::from([("a", 1)]); + let mut payload = Payload::from_map(data.clone()); + payload.insert("b", 2); + assert_eq!(data, Map::from([("a", 1)])); + assert_eq!(payload.to_dict(), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn an_empty_payload_has_no_keys() { + assert_eq!(Payload::new().to_dict(), Map::new()); +} + +#[test] +fn get_gives_the_value_or_absent() { + let payload = Payload::from_map(Map::from([("a", Value::from(1)), ("n", Value::Null)])); + assert_eq!(payload.get("a"), Some(&Value::from(1))); + assert_eq!(payload.get("missing"), None); + // A default is the language's own: unwrap_or. A present null is not absent. + assert_eq!( + payload.get("missing").cloned().unwrap_or(Value::from(7)), + Value::from(7) + ); + assert_eq!( + payload.get("n").cloned().unwrap_or(Value::from(7)), + Value::Null + ); +} + +#[test] +fn has_is_true_for_a_key_with_a_null_value() { + let payload = Payload::from_map(Map::from([("n", Value::Null)])); + assert!(payload.has("n")); + assert!(!payload.has("missing")); +} + +#[test] +fn insert_remove_merge_and_with_trace_change_the_payload_and_return_it() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + let other = Payload::from_map(Map::from([("c", 3)])); + let returned: *const Payload = payload.insert("b", 2); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.remove("a"); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.merge(&other); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.with_trace("t-1"); + assert!(std::ptr::eq(returned, &payload)); + assert_eq!(payload.to_dict(), Map::from([("b", 2), ("c", 3)])); + assert_eq!(payload.trace_id(), Some("t-1")); +} + +#[test] +fn writes_chain() { + let mut payload = Payload::new(); + payload.insert("a", 1).insert("b", 2).remove("a"); + assert_eq!(payload.to_dict(), Map::from([("b", 2)])); +} + +#[test] +fn remove_of_a_key_that_is_not_there_does_nothing() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + payload.remove("zz"); + assert_eq!(payload.to_dict(), Map::from([("a", 1)])); +} + +#[test] +fn in_a_merge_the_other_payload_wins() { + let mut payload = Payload::from_map(Map::from([("a", 1), ("b", 1)])); + payload.merge(&Payload::from_map(Map::from([("b", 2)]))); + assert_eq!(payload.to_dict(), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn a_merge_keeps_the_trace_id_it_has_and_takes_one_when_it_has_none() { + let mut other = Payload::new(); + other.with_trace("other"); + + let mut mine = Payload::new(); + mine.with_trace("mine").merge(&other); + assert_eq!(mine.trace_id(), Some("mine")); + + let mut none = Payload::new(); + none.merge(&other); + assert_eq!(none.trace_id(), Some("other")); +} + +#[test] +fn copy_gives_a_separate_payload_with_the_same_contents() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + payload.with_trace("t-1"); + let mut copy = payload.copy(); + copy.insert("a", 2); + assert_eq!(payload.get("a"), Some(&Value::from(1))); + assert_eq!(copy.trace_id(), Some("t-1")); +} + +#[test] +fn a_nested_value_changed_in_a_copy_is_not_changed_in_the_original() { + // D6: copy() is deep in Rust. A shared value could not be changed in + // place anyway, so the behaviour is the same as a shallow copy. + let payload = Payload::from_map(Map::from([("list", Value::List(vec![Value::from(1)]))])); + let mut copy = payload.copy(); + let mut list = copy + .get("list") + .and_then(Value::as_list) + .unwrap_or(&[]) + .to_vec(); + list.push(Value::from(2)); + copy.insert("list", list); + assert_eq!( + payload.get("list"), + Some(&Value::List(vec![Value::from(1)])) + ); +} + +#[test] +fn to_dict_gives_a_new_map_each_time() { + let payload = Payload::from_map(Map::from([("a", 1)])); + let mut out = payload.to_dict(); + out.insert("a", 2); + assert_eq!(payload.get("a"), Some(&Value::from(1))); +} + +#[test] +fn the_trace_id_is_absent_until_it_is_set_and_the_lineage_starts_empty() { + let payload = Payload::new(); + assert_eq!(payload.trace_id(), None); + assert!(payload.lineage().is_empty()); +} + +#[test] +fn keys_keep_the_order_in_which_they_were_inserted() { + let mut payload = Payload::new(); + payload + .insert("b", 1) + .insert("a", 2) + .insert("c", 3) + .remove("a"); + payload.insert("a", 4); + let out = payload.to_dict(); + assert_eq!(out.keys().collect::>(), ["b", "c", "a"]); +} + +#[test] +fn numbers_compare_as_json_numbers_and_true_is_not_one() { + assert_eq!(Value::from(1), Value::from(1.0)); + assert_ne!(Value::from(true), Value::from(1)); +} + +#[test] +fn maps_with_the_same_entries_are_equal_in_any_order() { + assert_eq!( + Map::from([("a", 1), ("b", 2)]), + Map::from([("b", 2), ("a", 1)]) + ); + assert_ne!(Map::from([("a", 1)]), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn a_payload_can_go_to_another_thread() { + fn send_and_sync() {} + send_and_sync::(); + send_and_sync::(); +} diff --git a/packages/codeupipe-rs/tests/pipeline.rs b/packages/codeupipe-rs/tests/pipeline.rs new file mode 100644 index 0000000..3f2c5e6 --- /dev/null +++ b/packages/codeupipe-rs/tests/pipeline.rs @@ -0,0 +1,463 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{ + Filter, Hook, Map, NamedPipeline, Payload, Pipeline, Predicate, StepError, Tap, Value, +}; + +fn n(payload: &Payload) -> f64 { + payload.get("n").and_then(Value::as_f64).unwrap_or(0.0) +} + +struct AddFilter; + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("n", n(payload) + 1.0); + Ok(()) + } +} + +struct AddOtherFilter; + +impl Filter for AddOtherFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("n", n(payload) + 1.0); + Ok(()) + } +} + +struct FailFilter; + +impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err("boom".into()) + } +} + +/// Writes, then fails: a Hook sees the Payload as the step left it. +struct HalfDoneFilter; + +impl Filter for HalfDoneFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("half", true); + Err("stopped half way".into()) + } +} + +struct IsOnPredicate; + +impl Predicate for IsOnPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("on") == Some(&Value::Bool(true)) + } +} + +struct SeeTap { + seen: Rc>>>, +} + +impl Tap for SeeTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + self.seen.borrow_mut().push(payload.get("n").cloned()); + Ok(()) + } +} + +struct FailTap; + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink is down".into()) + } +} + +struct FailOtherTap; + +impl Tap for FailOtherTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink is down again".into()) + } +} + +struct InnerPipeline; + +impl NamedPipeline for InnerPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + } +} + +#[derive(Clone, Default)] +struct RecordHook { + events: Rc>>, +} + +impl RecordHook { + fn events(&self) -> Vec { + self.events.borrow().clone() + } +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("before:{}", step.unwrap_or(""))); + } + fn after(&self, step: Option<&str>, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("after:{}", step.unwrap_or(""))); + } + fn on_error(&self, step: Option<&str>, _error: &StepError, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("error:{}", step.unwrap_or(""))); + } +} + +fn labels_of_errors(pipeline: &Pipeline) -> Vec { + pipeline + .state() + .errors() + .iter() + .map(|(label, _)| label.clone()) + .collect() +} + +#[test] +fn an_empty_pipeline_returns_the_data_it_was_given() { + let out = Pipeline::new() + .run(&Payload::from_map(Map::from([("a", 1)]))) + .unwrap(); + assert_eq!(out.to_dict(), Map::from([("a", 1)])); +} + +#[test] +fn steps_run_in_order_and_state_records_them() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddOtherFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 2.0); + assert_eq!(pipeline.state().executed(), ["AddFilter", "AddOtherFilter"]); + assert!(pipeline.state().skipped().is_empty()); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn run_works_on_a_copy_so_the_callers_payload_is_not_changed() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + let given = Payload::from_map(Map::from([("n", 1)])); + let out = pipeline.run(&given).unwrap(); + assert_eq!(n(&given), 1.0); + assert_eq!(n(&out), 2.0); +} + +#[test] +fn a_valve_runs_its_filter_when_the_predicate_is_true() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + let out = pipeline + .run(&Payload::from_map(Map::from([("on", true)]))) + .unwrap(); + assert_eq!(n(&out), 1.0); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); + assert!(pipeline.state().skipped().is_empty()); +} + +#[test] +fn a_valve_skips_its_filter_when_the_predicate_is_false() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + let out = pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + assert!(!out.has("n")); + assert!(pipeline.state().executed().is_empty()); + assert_eq!(pipeline.state().skipped(), ["AddFilter"]); +} + +#[test] +fn a_filter_failure_stops_the_run_and_state_keeps_the_steps_before_it() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(FailFilter); + pipeline.add_filter(AddFilter); + let error = pipeline.run(&Payload::new()).unwrap_err(); + assert_eq!(error.to_string(), "boom"); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); +} + +#[test] +fn a_tap_reads_the_payload_and_counts_as_executed() { + let seen = Rc::new(RefCell::new(Vec::new())); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap { seen: seen.clone() }); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!(*seen.borrow(), [Some(Value::from(1))]); + assert_eq!(pipeline.state().executed(), ["AddFilter", "SeeTap"]); +} + +#[test] +fn a_tap_failure_is_recorded_and_does_not_stop_the_run() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_tap(FailTap); + pipeline.add_tap(FailOtherTap); + pipeline.add_filter(AddFilter); + pipeline.use_hook(hook.clone()); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 1.0); + assert_eq!( + pipeline.state().executed(), + ["FailTap", "FailOtherTap", "AddFilter"] + ); + assert_eq!(labels_of_errors(&pipeline), ["FailTap", "FailOtherTap"]); + assert_eq!(pipeline.state().errors()[0].1.to_string(), "sink is down"); + assert_eq!( + hook.events(), + ["before:", "before:AddFilter", "after:AddFilter", "after:"] + ); +} + +#[test] +fn a_nested_pipeline_is_one_step() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); + pipeline.add_filter(AddFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 3.0); + assert_eq!(pipeline.state().executed(), ["InnerPipeline", "AddFilter"]); +} + +#[test] +fn hooks_see_the_pipeline_and_each_filter_or_valve_in_order() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_valve(IsOnPredicate, AddOtherFilter); + pipeline.add_tap(SeeTap { + seen: Rc::default(), + }); + pipeline.use_hook(hook.clone()); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!( + hook.events(), + [ + "before:", + "before:AddFilter", + "after:AddFilter", + "before:AddOtherFilter", + "after:AddOtherFilter", + "after:" + ] + ); +} + +#[test] +fn on_a_filter_failure_a_hook_gets_the_step_then_the_pipeline_and_no_after() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(FailFilter); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + assert_eq!( + hook.events(), + [ + "before:", + "before:AddFilter", + "after:AddFilter", + "before:FailFilter", + "error:FailFilter", + "error:" + ] + ); +} + +#[test] +fn a_hook_gets_the_error_itself() { + #[derive(Clone, Default)] + struct KeepErrorHook { + seen: Rc>>, + } + impl Hook for KeepErrorHook { + fn on_error(&self, _step: Option<&str>, error: &StepError, _payload: &Payload) { + self.seen.borrow_mut().push(address(error)); + } + } + fn address(error: &StepError) -> usize { + &**error as *const (dyn std::error::Error + Send + Sync) as *const () as usize + } + + let hook = KeepErrorHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(FailFilter); + pipeline.use_hook(hook.clone()); + let error = pipeline.run(&Payload::new()).unwrap_err(); + // Both calls get the error that run returns: the same value, not a copy. + assert_eq!(*hook.seen.borrow(), [address(&error), address(&error)]); +} + +#[test] +fn on_error_gets_the_payload_as_the_failing_step_left_it() { + #[derive(Clone, Default)] + struct KeepPayloadHook { + seen: Rc>>, + } + impl Hook for KeepPayloadHook { + fn on_error(&self, _step: Option<&str>, _error: &StepError, payload: &Payload) { + self.seen.borrow_mut().push(payload.to_dict()); + } + } + + let hook = KeepPayloadHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(HalfDoneFilter); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + let expected = Map::from([("n", Value::from(1)), ("half", Value::from(true))]); + assert_eq!(*hook.seen.borrow(), [expected.clone(), expected]); +} + +#[test] +fn timing_and_lineage_are_off_by_default() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert!(pipeline.state().timings().is_empty()); + assert!(out.lineage().is_empty()); +} + +#[test] +fn observe_turns_timing_on_and_lineage_when_asked() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap { + seen: Rc::default(), + }); + pipeline.add_valve(IsOnPredicate, AddOtherFilter); + pipeline.observe(true, true); + let out = pipeline.run(&Payload::new()).unwrap(); + let state = pipeline.state(); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["AddFilter", "AddOtherFilter"]); + assert_eq!(out.lineage(), ["AddFilter", "AddOtherFilter"]); +} + +#[test] +fn each_run_starts_with_a_new_state() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + pipeline + .run(&Payload::from_map(Map::from([("on", true)]))) + .unwrap(); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); + assert!(pipeline.state().skipped().is_empty()); +} + +#[test] +fn a_shorter_run_after_a_longer_one_keeps_nothing_of_it() { + struct FailIfAskedFilter; + impl Filter for FailIfAskedFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("fail") == Some(&Value::Bool(true)) { + return Err("asked to fail".into()); + } + Ok(()) + } + } + struct FailIfAskedTap; + impl Tap for FailIfAskedTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + if payload.get("tap_fail") == Some(&Value::Bool(true)) { + return Err("asked to fail".into()); + } + Ok(()) + } + } + let mut pipeline = Pipeline::new(); + pipeline.observe(true, false); + pipeline.add_filter(FailIfAskedFilter); + pipeline.add_valve(IsOnPredicate, AddFilter); + pipeline.add_tap(FailIfAskedTap); + + pipeline + .run(&Payload::from_map(Map::from([ + ("on", true), + ("tap_fail", true), + ]))) + .unwrap(); + let state = pipeline.state(); + assert_eq!( + state.executed(), + ["FailIfAskedFilter", "AddFilter", "FailIfAskedTap"] + ); + assert_eq!(state.errors().len(), 1); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["FailIfAskedFilter", "AddFilter"]); + + pipeline + .run(&Payload::from_map(Map::from([("fail", true)]))) + .unwrap_err(); + let state = pipeline.state(); + assert!(state.executed().is_empty()); + assert!(state.skipped().is_empty()); + assert!(state.errors().is_empty()); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["FailIfAskedFilter"]); + + pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + let state = pipeline.state(); + assert_eq!(state.executed(), ["FailIfAskedFilter", "FailIfAskedTap"]); + assert_eq!(state.skipped(), ["AddFilter"]); + assert!(state.errors().is_empty()); +} + +#[test] +fn a_failure_inside_a_nested_pipeline_is_reported_as_the_nesting_step() { + struct FailingInnerPipeline; + impl NamedPipeline for FailingInnerPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(FailFilter); + } + } + + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(FailingInnerPipeline)); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + assert_eq!( + hook.events(), + [ + "before:", + "before:FailingInnerPipeline", + "error:FailingInnerPipeline", + "error:" + ] + ); +} diff --git a/packages/codeupipe-rs/tests/pricing_example.rs b/packages/codeupipe-rs/tests/pricing_example.rs new file mode 100644 index 0000000..faa3653 --- /dev/null +++ b/packages/codeupipe-rs/tests/pricing_example.rs @@ -0,0 +1,59 @@ +// The example's step files, as modules of this test. +#[path = "../examples/pricing/at_least_predicate.rs"] +mod at_least_predicate; +#[path = "../examples/pricing/audit_tap.rs"] +mod audit_tap; +#[path = "../examples/pricing/discount_filter.rs"] +mod discount_filter; +#[path = "../examples/pricing/mark_bulk_filter.rs"] +mod mark_bulk_filter; +#[path = "../examples/pricing/pricing_pipeline.rs"] +mod pricing_pipeline; + +use codeupipe::{Map, Payload, Pipeline, Value}; +use pricing_pipeline::PricingPipeline; + +#[test] +fn a_large_order_gets_the_discount_and_the_bulk_mark() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline + .run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 12.0), + ]))) + .unwrap(); + assert_eq!( + out.to_dict(), + Map::from([ + ("price", Value::from(90)), + ("qty", Value::from(12)), + ("bulk", Value::from(true)) + ]) + ); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + ); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn the_tap_reads_the_payload() { + use codeupipe::Tap; + assert!(audit_tap::AuditTap + .observe(&Payload::from_map(Map::from([("price", 90)]))) + .is_ok()); +} + +#[test] +fn a_small_order_gets_the_discount_only() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline + .run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 2.0), + ]))) + .unwrap(); + assert_eq!(out.to_dict(), Map::from([("price", 90), ("qty", 2)])); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} diff --git a/packages/codeupipe-rs/tests/state_per_run.rs b/packages/codeupipe-rs/tests/state_per_run.rs new file mode 100644 index 0000000..9a538c4 --- /dev/null +++ b/packages/codeupipe-rs/tests/state_per_run.rs @@ -0,0 +1,159 @@ +// A State belongs to one run, not to the Pipeline (conformance/API.md, +// "State"): the caller can give the State, each run writes into its own, a +// step can run its own pipeline again while it runs, and state() is the +// State of the run that started last. +use std::cell::RefCell; +use std::rc::{Rc, Weak}; + +use codeupipe::{Filter, Map, Payload, Pipeline, Predicate, State, StepError, Tap, Value}; + +struct SetFilter; + +impl Filter for SetFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("a", 1); + Ok(()) + } +} + +struct IsInnerPredicate; + +impl Predicate for IsInnerPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("inner") == Some(&Value::Bool(true)) + } +} + +struct MarkFilter; + +impl Filter for MarkFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("gated", true); + Ok(()) + } +} + +struct FailTap; + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink down".into()) + } +} + +/// Runs the pipeline that it is in one more time, with `inner` set, and a +/// State of its own. The pipeline is given to it after it is made. +#[derive(Clone, Default)] +struct RunAgainFilter { + pipeline: Rc>>, + inner_states: Rc>>, +} + +impl Filter for RunAgainFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("inner") == Some(&Value::Bool(true)) { + return Ok(()); + } + let pipeline = self + .pipeline + .borrow() + .upgrade() + .expect("the pipeline is there"); + let mut state = State::new(); + pipeline.run_with_state(&Payload::from_map(Map::from([("inner", true)])), &mut state)?; + self.inner_states.borrow_mut().push(state); + Ok(()) + } +} + +/// first, again (a run inside the run), gate (only for the inner run). +fn pipeline_that_runs_again() -> (Rc, RunAgainFilter) { + let again = RunAgainFilter::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_filter(again.clone()); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + let pipeline = Rc::new(pipeline); + *again.pipeline.borrow_mut() = Rc::downgrade(&pipeline); + (pipeline, again) +} + +#[test] +fn the_caller_gives_the_state_of_a_run() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + pipeline.add_tap(FailTap); + let mut state = State::new(); + let out = pipeline + .run_with_state(&Payload::new(), &mut state) + .unwrap(); + assert_eq!(out.get("a"), Some(&Value::from(1))); + assert_eq!(state.executed(), ["SetFilter", "FailTap"]); + assert_eq!(state.skipped(), ["MarkFilter"]); + assert_eq!(state.errors()[0].1.to_string(), "sink down"); + // It is also the State of the run that started last. + assert_eq!(pipeline.state().executed(), state.executed()); + assert_eq!(pipeline.state().errors()[0].1.to_string(), "sink down"); +} + +#[test] +fn a_run_that_starts_inside_another_run_has_its_own_state() { + let (pipeline, again) = pipeline_that_runs_again(); + let mut outer = State::new(); + pipeline + .run_with_state(&Payload::new(), &mut outer) + .unwrap(); + assert_eq!(outer.executed(), ["SetFilter", "RunAgainFilter"]); + assert_eq!(outer.skipped(), ["MarkFilter"]); + let inner = again.inner_states.borrow(); + assert_eq!(inner.len(), 1); + assert_eq!( + inner[0].executed(), + ["SetFilter", "RunAgainFilter", "MarkFilter"] + ); + assert!(inner[0].skipped().is_empty()); +} + +#[test] +fn with_no_state_from_the_caller_the_pipeline_has_the_state_of_the_run_that_started_last() { + let (pipeline, _again) = pipeline_that_runs_again(); + pipeline.run(&Payload::new()).unwrap(); + // The inner run started last, inside the outer one. + assert_eq!( + pipeline.state().executed(), + ["SetFilter", "RunAgainFilter", "MarkFilter"] + ); +} + +#[test] +fn a_state_that_was_read_does_not_change_with_the_next_run() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + pipeline + .run(&Payload::from_map(Map::from([("inner", true)]))) + .unwrap(); + let first = pipeline.state(); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!(first.executed(), ["MarkFilter"]); + assert_eq!(pipeline.state().skipped(), ["MarkFilter"]); +} + +#[test] +fn a_failed_run_with_the_callers_state_keeps_the_steps_before_the_failure() { + struct FailFilter; + impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err("boom".into()) + } + } + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_filter(FailFilter); + let mut state = State::new(); + assert!(pipeline + .run_with_state(&Payload::new(), &mut state) + .is_err()); + assert_eq!(state.executed(), ["SetFilter"]); + assert_eq!(pipeline.state().executed(), ["SetFilter"]); +} diff --git a/packages/codeupipe-rs/tests/step_label.rs b/packages/codeupipe-rs/tests/step_label.rs new file mode 100644 index 0000000..ec6d9d9 --- /dev/null +++ b/packages/codeupipe-rs/tests/step_label.rs @@ -0,0 +1,110 @@ +use std::marker::PhantomData; + +use codeupipe::{Filter, NamedPipeline, Payload, Pipeline, Predicate, StepError, Tap, Value}; + +struct AddFilter; + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let n = payload.get("n").and_then(Value::as_f64).unwrap_or(0.0); + payload.insert("n", n + 1.0); + Ok(()) + } +} + +struct SeeTap; + +impl Tap for SeeTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Ok(()) + } +} + +struct IsOnPredicate; + +impl Predicate for IsOnPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("on") == Some(&Value::Bool(true)) + } +} + +struct InnerPipeline; + +impl NamedPipeline for InnerPipeline { + fn add_steps(&self, _pipeline: &mut Pipeline) {} +} + +#[test] +fn the_label_is_the_name_of_the_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap); + assert_eq!(pipeline.labels(), ["AddFilter", "SeeTap"]); +} + +#[test] +fn the_second_use_of_a_type_gets_a_number() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + assert_eq!( + pipeline.labels(), + ["AddFilter", "AddFilter#2", "AddFilter#3"] + ); +} + +#[test] +fn a_valve_has_the_label_of_its_filter() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_valve(IsOnPredicate, AddFilter); + assert_eq!(pipeline.labels(), ["AddFilter", "AddFilter#2"]); +} + +#[test] +fn a_nested_pipeline_has_the_label_of_its_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); + pipeline.add_pipeline(Pipeline::new()); + assert_eq!(pipeline.labels(), ["InnerPipeline", "Pipeline"]); +} + +#[test] +fn a_type_defined_in_a_function_works() { + fn make() -> impl Filter { + struct LocalFilter; + impl Filter for LocalFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Ok(()) + } + } + LocalFilter + } + let mut pipeline = Pipeline::new(); + pipeline.add_filter(make()); + assert_eq!(pipeline.labels(), ["LocalFilter"]); +} + +#[test] +fn a_generic_type_has_its_name_without_the_arguments() { + struct WrapFilter(PhantomData); + impl Filter for WrapFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Ok(()) + } + } + let mut pipeline = Pipeline::new(); + pipeline.add_filter(WrapFilter::>>( + PhantomData, + )); + assert_eq!(pipeline.labels(), ["WrapFilter"]); +} + +#[test] +fn labels_gives_a_new_list_each_time() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.labels().push("x".to_string()); + assert_eq!(pipeline.labels(), ["AddFilter"]); +} diff --git a/packages/codeupipe-rs/tests/typed_payload.rs b/packages/codeupipe-rs/tests/typed_payload.rs new file mode 100644 index 0000000..8786724 --- /dev/null +++ b/packages/codeupipe-rs/tests/typed_payload.rs @@ -0,0 +1,207 @@ +// A pipeline over a type of your own. Payload is the default, a map that any +// developer can read; a struct can take its place, and then every step of +// that pipeline takes the struct. +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Filter, Hook, NamedPipeline, Pipeline, PipelineData, Predicate, StepError, Tap}; + +#[derive(Clone, Debug, Default, PartialEq)] +struct Order { + price: f64, + qty: u32, + bulk: bool, +} + +impl PipelineData for Order {} + +struct DiscountFilter { + rate: f64, +} + +impl Filter for DiscountFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.price = (order.price * (1.0 - self.rate) * 100.0).round() / 100.0; + Ok(()) + } +} + +struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} + +struct RefuseFreeFilter; + +impl Filter for RefuseFreeFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + if order.price <= 0.0 { + return Err("an order must cost something".into()); + } + Ok(()) + } +} + +struct AtLeastPredicate { + minimum: u32, +} + +impl Predicate for AtLeastPredicate { + fn test(&self, order: &Order) -> bool { + order.qty >= self.minimum + } +} + +struct SeeTap { + seen: Rc>>, +} + +impl Tap for SeeTap { + fn observe(&self, order: &Order) -> Result<(), StepError> { + self.seen.borrow_mut().push(order.price); + Ok(()) + } +} + +#[derive(Clone, Default)] +struct RecordHook { + events: Rc>>, +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _order: &Order) { + self.events + .borrow_mut() + .push(format!("before:{}", step.unwrap_or(""))); + } + fn on_error(&self, step: Option<&str>, _error: &StepError, order: &Order) { + self.events + .borrow_mut() + .push(format!("error:{}:{}", step.unwrap_or(""), order.price)); + } +} + +struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter { rate: 0.1 }); + pipeline.add_valve(AtLeastPredicate { minimum: 10 }, MarkBulkFilter); + } +} + +fn order(price: f64, qty: u32) -> Order { + Order { + price, + qty, + bulk: false, + } +} + +#[test] +fn every_primitive_takes_the_type() { + let seen = Rc::new(RefCell::new(Vec::new())); + let hook = RecordHook::default(); + let mut pipeline: Pipeline = Pipeline::new(); + pipeline.add_filter(DiscountFilter { rate: 0.1 }); + pipeline.add_valve(AtLeastPredicate { minimum: 10 }, MarkBulkFilter); + pipeline.add_tap(SeeTap { + seen: Rc::clone(&seen), + }); + pipeline.use_hook(hook.clone()); + + let input = order(100.0, 12); + let out = pipeline.run(&input).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 12, + bulk: true + } + ); + assert_eq!(input, order(100.0, 12)); // the caller's value is not changed + assert_eq!(*seen.borrow(), [90.0]); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "SeeTap"] + ); + assert_eq!( + *hook.events.borrow(), + ["before:", "before:DiscountFilter", "before:MarkBulkFilter"] + ); + + pipeline.run(&order(100.0, 2)).unwrap(); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} + +#[test] +fn a_named_and_nested_pipeline_takes_the_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(PricingPipeline)); + pipeline.add_filter(RefuseFreeFilter); + let out = pipeline.run(&order(50.0, 10)).unwrap(); + assert_eq!( + out, + Order { + price: 45.0, + qty: 10, + bulk: true + } + ); + assert_eq!(pipeline.labels(), ["PricingPipeline", "RefuseFreeFilter"]); +} + +#[test] +fn a_failure_is_the_same_as_with_a_payload() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(DiscountFilter { rate: 1.0 }); + pipeline.add_filter(RefuseFreeFilter); + pipeline.use_hook(hook.clone()); + let error = pipeline.run(&order(10.0, 1)).unwrap_err(); + assert_eq!(error.to_string(), "an order must cost something"); + assert_eq!(pipeline.state().executed(), ["DiscountFilter"]); + let events = hook.events.borrow(); + assert_eq!( + events[events.len() - 2..], + ["error:RefuseFreeFilter:0", "error::0"] + ); +} + +#[test] +fn lineage_is_kept_only_by_a_type_that_keeps_it() { + #[derive(Clone, Default)] + struct Traced { + steps: Vec, + } + impl PipelineData for Traced { + fn stamp(&mut self, label: &str) { + self.steps.push(label.to_string()); + } + } + struct NoopFilter; + impl Filter for NoopFilter { + fn call(&self, _traced: &mut Traced) -> Result<(), StepError> { + Ok(()) + } + } + let mut traced = Pipeline::new(); + traced.add_filter(NoopFilter); + traced.observe(false, true); + assert_eq!( + traced.run(&Traced::default()).unwrap().steps, + ["NoopFilter"] + ); + + // Order does not keep lineage: with it on, the run is the same. + let mut plain = Pipeline::new(); + plain.add_filter(MarkBulkFilter); + plain.observe(true, true); + assert!(plain.run(&order(1.0, 1)).unwrap().bulk); + assert_eq!(plain.state().timings().len(), 1); +} diff --git a/packages/codeupipe-rs/tests/typed_pricing_example.rs b/packages/codeupipe-rs/tests/typed_pricing_example.rs new file mode 100644 index 0000000..35479cf --- /dev/null +++ b/packages/codeupipe-rs/tests/typed_pricing_example.rs @@ -0,0 +1,53 @@ +// The typed example's files, as modules of this test. It is the pricing +// example with an Order struct in place of the Payload, and it must give the +// same result. +#[path = "../examples/typed_pricing/at_least_predicate.rs"] +mod at_least_predicate; +#[path = "../examples/typed_pricing/audit_tap.rs"] +mod audit_tap; +#[path = "../examples/typed_pricing/discount_filter.rs"] +mod discount_filter; +#[path = "../examples/typed_pricing/mark_bulk_filter.rs"] +mod mark_bulk_filter; +#[path = "../examples/typed_pricing/order.rs"] +mod order; +#[path = "../examples/typed_pricing/pricing_pipeline.rs"] +mod pricing_pipeline; + +use codeupipe::Pipeline; +use order::Order; +use pricing_pipeline::PricingPipeline; + +#[test] +fn a_large_order_gets_the_discount_and_the_bulk_mark() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline.run(&Order::new(100.0, 12)).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 12, + bulk: true + } + ); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + ); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn a_small_order_gets_the_discount_only() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline.run(&Order::new(100.0, 2)).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 2, + bulk: false + } + ); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} diff --git a/packages/codeupipe-ts/README.md b/packages/codeupipe-ts/README.md index e2a47a3..a92d274 100644 --- a/packages/codeupipe-ts/README.md +++ b/packages/codeupipe-ts/README.md @@ -99,7 +99,7 @@ export class PricingPipeline extends Pipeline { this.addFilter(new DiscountFilter(0.1)); // MarkBulkFilter runs only when the predicate is true. this.addValve(new AtLeastPredicate("qty", 10), new MarkBulkFilter()); - // A Tap failure is recorded and does not stop the run. + // A Tap only reads. If it fails, State records the error and the run goes on. this.addTap(new AuditTap()); } } @@ -115,6 +115,36 @@ result.toDict(); In this repository the example imports from `../../src/index.ts`. +## A type of your own + +`Payload` is the default: a map that any developer can read. A class of your +own can take its place. It needs `copy()`, and the steps of that pipeline +take it: + +```ts +class Order implements PipelineData { + price: number; + constructor(price: number) { this.price = price; } + copy(): Order { return new Order(this.price); } +} + +class DiscountFilter implements Filter { + call(order: Order): Order { + order.price = Math.round(order.price * 90) / 100; + return order; + } +} + +const pipeline = new Pipeline(); +pipeline.addFilter(new DiscountFilter()); +``` + +A step for another type does not type-check. Lineage is kept only if the +class has `_stamp(label)`; otherwise `observe({ lineage: true })` does +nothing for it. The compiler reads Payload operations only, so it refuses a +step that uses `Order`. See "A type of your own" in +[conformance/API.md](../../conformance/API.md). + ## Names The names are the names of the contract, in camelCase. diff --git a/packages/codeupipe-ts/examples/pricing/audit-tap.ts b/packages/codeupipe-ts/examples/pricing/audit-tap.ts index 5ba2343..35671c3 100644 --- a/packages/codeupipe-ts/examples/pricing/audit-tap.ts +++ b/packages/codeupipe-ts/examples/pricing/audit-tap.ts @@ -1,11 +1,11 @@ import type { Payload, Tap } from "../../src/index.ts"; /** - * A Tap: reads the Payload and returns nothing. This one fails every time, - * to show that a Tap failure is recorded and never stops the run. + * A Tap: reads the Payload and returns nothing. If it fails, State records + * the error and the run goes on. */ export class AuditTap implements Tap { - observe(_payload: Payload): void { - throw new Error("audit sink is down"); + observe(payload: Payload): void { + console.log("audit:", payload.toDict()); } } diff --git a/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts b/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts index 6b8a617..ab40e84 100644 --- a/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts +++ b/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts @@ -11,7 +11,7 @@ export class PricingPipeline extends Pipeline { this.addFilter(new DiscountFilter(0.1)); // MarkBulkFilter runs only when the predicate is true. this.addValve(new AtLeastPredicate("qty", 10), new MarkBulkFilter()); - // A Tap failure is recorded and does not stop the run. + // A Tap only reads. If it fails, State records the error and the run goes on. this.addTap(new AuditTap()); } } diff --git a/packages/codeupipe-ts/src/filter.ts b/packages/codeupipe-ts/src/filter.ts index 9b130b2..f621232 100644 --- a/packages/codeupipe-ts/src/filter.ts +++ b/packages/codeupipe-ts/src/filter.ts @@ -1,6 +1,6 @@ import type { Payload } from "./payload.ts"; /** Filter: changes the data. A failure throws. `call` can be async. */ -export interface Filter { - call(payload: Payload): Payload | Promise; +export interface Filter

{ + call(payload: P): P | Promise

; } diff --git a/packages/codeupipe-ts/src/hook.ts b/packages/codeupipe-ts/src/hook.ts index 6303679..008b5ea 100644 --- a/packages/codeupipe-ts/src/hook.ts +++ b/packages/codeupipe-ts/src/hook.ts @@ -6,13 +6,13 @@ import type { Payload } from "./payload.ts"; * `step` is the label of the step, or null for the pipeline itself. A Hook * reads the Payload and does not write to it. Its methods can be async. */ -export class Hook { +export class Hook

{ /** Before a step, or before the pipeline starts (`step` is null). */ - before(step: string | null, payload: Payload): void | Promise {} + before(step: string | null, payload: P): void | Promise {} /** After a step, or after the pipeline ends (`step` is null). */ - after(step: string | null, payload: Payload): void | Promise {} + after(step: string | null, payload: P): void | Promise {} /** When a Filter fails: for the step, then for the pipeline (`step` is null). */ - onError(step: string | null, error: unknown, payload: Payload): void | Promise {} + onError(step: string | null, error: unknown, payload: P): void | Promise {} } diff --git a/packages/codeupipe-ts/src/index.ts b/packages/codeupipe-ts/src/index.ts index d1a5888..093c7fa 100644 --- a/packages/codeupipe-ts/src/index.ts +++ b/packages/codeupipe-ts/src/index.ts @@ -9,6 +9,7 @@ export type { Filter } from "./filter.ts"; export { Hook } from "./hook.ts"; export { Payload } from "./payload.ts"; export { Pipeline } from "./pipeline.ts"; +export type { PipelineData } from "./pipeline-data.ts"; export type { Predicate } from "./predicate.ts"; export { State } from "./state.ts"; export type { Tap } from "./tap.ts"; diff --git a/packages/codeupipe-ts/src/pipeline-data.ts b/packages/codeupipe-ts/src/pipeline-data.ts new file mode 100644 index 0000000..543e96a --- /dev/null +++ b/packages/codeupipe-ts/src/pipeline-data.ts @@ -0,0 +1,24 @@ +/** + * What a Pipeline carries from step to step. + * + * `Payload` is the default: a map that any developer can read, and that + * every runtime of codeupipe has. A class of your own can take its place, + * such as the class that your steps would use anyway: + * + * class Order implements PipelineData { + * copy(): Order { ... } + * } + * class DiscountFilter implements Filter { ... } + * const pipeline = new Pipeline(); + * + * The Pipeline asks one thing of it, `copy()`, for the copy that a run works + * on. The Filters, Predicates, Taps and Hooks of that pipeline then take the + * class, and a step for another type does not type-check. + * + * Lineage (`observe({ lineage: true })`) is kept only by a type that has + * `_stamp(label)`, as Payload has. For any other type it does nothing. + */ +export interface PipelineData

{ + copy(): P; + _stamp?(label: string): unknown; +} diff --git a/packages/codeupipe-ts/src/pipeline.ts b/packages/codeupipe-ts/src/pipeline.ts index b5bd23a..a9eeaa3 100644 --- a/packages/codeupipe-ts/src/pipeline.ts +++ b/packages/codeupipe-ts/src/pipeline.ts @@ -1,17 +1,18 @@ import type { Filter } from "./filter.ts"; import type { Hook } from "./hook.ts"; import type { Payload } from "./payload.ts"; +import type { PipelineData } from "./pipeline-data.ts"; import type { Predicate } from "./predicate.ts"; import { State } from "./state.ts"; import { labelOf } from "./step-label.ts"; import type { Tap } from "./tap.ts"; import { Valve } from "./valve.ts"; -type Step = - | { kind: "filter"; label: string; filter: Filter } - | { kind: "valve"; label: string; valve: Valve } - | { kind: "tap"; label: string; tap: Tap } - | { kind: "pipeline"; label: string; pipeline: Pipeline }; +type Step

> = + | { kind: "filter"; label: string; filter: Filter

} + | { kind: "valve"; label: string; valve: Valve

} + | { kind: "tap"; label: string; tap: Tap

} + | { kind: "pipeline"; label: string; pipeline: Pipeline

}; // The clock of browsers and of Node. Declared here so that this file needs // the types of neither. @@ -43,11 +44,14 @@ function require(step: unknown, method: string, what: string): void { // cup: al * lifecycle, and State records what ran. Add steps with `addFilter`, * `addValve`, `addTap` and `addPipeline`, add Hooks with `useHook`, and run * with `await pipeline.run(payload)`. + * + * It carries a Payload unless you give it a type of your own + * (`new Pipeline()`, see `PipelineData`). */ -export class Pipeline { - #steps: Step[] = []; +export class Pipeline

= Payload> { + #steps: Step

[] = []; #uses: Map = new Map(); - #hooks: Hook[] = []; + #hooks: Hook

[] = []; #state: State = new State(); #timing = false; #lineage = false; @@ -71,7 +75,7 @@ export class Pipeline { } /** Add a Filter. */ - addFilter(filter: Filter): void { + addFilter(filter: Filter

): void { require(filter, "call", "A Filter"); this.#steps.push({ kind: "filter", label: this.#label(filter), filter }); } @@ -83,7 +87,7 @@ export class Pipeline { * test is false the Payload passes through unchanged and the step is * recorded as skipped. The step has the label of its Filter. */ - addValve(predicate: Predicate, filter: Filter): void { + addValve(predicate: Predicate

, filter: Filter

): void { if (!has(predicate, "test") && has(predicate, "call") && has(filter, "test")) { throw new TypeError("The arguments are in the wrong order: addValve(predicate, filter). " + "The condition comes first, then the Filter."); @@ -94,13 +98,13 @@ export class Pipeline { } /** Add a Tap. */ - addTap(tap: Tap): void { + addTap(tap: Tap

): void { require(tap, "observe", "A Tap"); this.#steps.push({ kind: "tap", label: this.#label(tap), tap }); } /** Nest a Pipeline as one step of this Pipeline. */ - addPipeline(pipeline: Pipeline): void { + addPipeline(pipeline: Pipeline

): void { if (!(pipeline instanceof Pipeline)) { throw new TypeError("addPipeline takes a Pipeline. The value that was given is not one."); } @@ -108,11 +112,14 @@ export class Pipeline { } /** Add a Hook. */ - useHook(hook: Hook): void { + useHook(hook: Hook

): void { this.#hooks.push(hook); } - /** Turn on timing (seconds for each step, in State) and lineage (labels on the Payload). */ + /** + * Turn on timing (seconds for each step, in State) and lineage (labels on + * the Payload). A type of your own keeps lineage only if it has `_stamp`. + */ observe(options: { timing?: boolean; lineage?: boolean } = {}): void { this.#timing = options.timing ?? true; this.#lineage = options.lineage ?? false; @@ -129,12 +136,12 @@ export class Pipeline { * State when not. A State belongs to one run: a caller with more than one * run of this Pipeline at a time gives each run its own. */ - async run(initial: Payload, state: State = new State()): Promise { + async run(initial: P, state: State = new State()): Promise

{ // The State is a local of this call, so two runs at the same time do not mix. this.#state = state; // the State of the run that started last const hooks = this.#hooks; const timed = this.#timing; - let payload = initial.copy(); // the caller's Payload is not changed + let payload = initial.copy(); // the caller's value is not changed for (const hook of hooks) { const done = hook.before(null, payload); @@ -184,7 +191,7 @@ export class Pipeline { if (isThenable(done)) await done; } if (timed && started !== null) state.recordTiming(label, (performance.now() - started) / 1000); - if (this.#lineage) payload._stamp(label); + if (this.#lineage) payload._stamp?.(label); } } catch (error) { if (timed && started !== null && failing !== null) { diff --git a/packages/codeupipe-ts/src/predicate.ts b/packages/codeupipe-ts/src/predicate.ts index 12336e8..321a99c 100644 --- a/packages/codeupipe-ts/src/predicate.ts +++ b/packages/codeupipe-ts/src/predicate.ts @@ -6,6 +6,6 @@ import type { Payload } from "./payload.ts"; * A Predicate is an object. Its settings (a threshold, a flag) are on the * instance. It reads the Payload and does not write to it. */ -export interface Predicate { - test(payload: Payload): boolean; +export interface Predicate

{ + test(payload: P): boolean; } diff --git a/packages/codeupipe-ts/src/tap.ts b/packages/codeupipe-ts/src/tap.ts index 2ccbd56..db841fd 100644 --- a/packages/codeupipe-ts/src/tap.ts +++ b/packages/codeupipe-ts/src/tap.ts @@ -4,6 +4,6 @@ import type { Payload } from "./payload.ts"; * Tap: reads the data as it goes by. It does not write to the Payload. * A Tap failure is recorded in State and never stops the run. */ -export interface Tap { - observe(payload: Payload): void | Promise; +export interface Tap

{ + observe(payload: P): void | Promise; } diff --git a/packages/codeupipe-ts/src/valve.ts b/packages/codeupipe-ts/src/valve.ts index 27d3038..ea4d335 100644 --- a/packages/codeupipe-ts/src/valve.ts +++ b/packages/codeupipe-ts/src/valve.ts @@ -7,11 +7,11 @@ import type { Predicate } from "./predicate.ts"; * * `Pipeline.addValve(predicate, filter)` makes one. It is not a public name. */ -export class Valve { - readonly predicate: Predicate; - readonly filter: Filter; +export class Valve

{ + readonly predicate: Predicate

; + readonly filter: Filter

; - constructor(predicate: Predicate, filter: Filter) { + constructor(predicate: Predicate

, filter: Filter

) { this.predicate = predicate; this.filter = filter; } diff --git a/packages/codeupipe-ts/tests/custom-payload.test.ts b/packages/codeupipe-ts/tests/custom-payload.test.ts new file mode 100644 index 0000000..afc27d1 --- /dev/null +++ b/packages/codeupipe-ts/tests/custom-payload.test.ts @@ -0,0 +1,177 @@ +// A pipeline over a type of your own. Payload is the default, a map that any +// developer can read; a class can take its place. The Pipeline asks one +// thing of it, copy(), for the copy that a run works on. Lineage is kept only +// by a type that has _stamp(label); for any other type it does nothing. +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { Hook, Payload, Pipeline } from "../src/index.ts"; +import type { Filter, PipelineData, Predicate, Tap } from "../src/index.ts"; + +class Order implements PipelineData { + price: number; + qty: number; + bulk: boolean; + + constructor(price: number, qty: number, bulk = false) { + this.price = price; + this.qty = qty; + this.bulk = bulk; + } + + copy(): Order { + return new Order(this.price, this.qty, this.bulk); + } +} + +class TracedOrder extends Order { + steps: string[] = []; + + override copy(): TracedOrder { + const copy = new TracedOrder(this.price, this.qty, this.bulk); + copy.steps = [...this.steps]; + return copy; + } + + _stamp(label: string): void { + this.steps.push(label); + } +} + +class DiscountFilter implements Filter { + call(order: Order): Order { + order.price = Math.round(order.price * 90) / 100; + return order; + } +} + +class MarkBulkFilter implements Filter { + call(order: Order): Order { + order.bulk = true; + return order; + } +} + +class RefuseFreeFilter implements Filter { + async call(order: Order): Promise { + if (order.price <= 0) throw new Error("an order must cost something"); + return order; + } +} + +class PassTracedFilter implements Filter { + call(order: TracedOrder): TracedOrder { + return order; + } +} + +class MarkRanFilter implements Filter { + call(payload: Payload): Payload { + return payload.insert("ran", true); + } +} + +class AtLeastPredicate implements Predicate { + test(order: Order): boolean { + return order.qty >= 10; + } +} + +class SeeTap implements Tap { + readonly seen: number[] = []; + + observe(order: Order): void { + this.seen.push(order.price); + } +} + +class RecordHook extends Hook { + readonly events: string[] = []; + + override before(step: string | null): void { + this.events.push(`before:${step ?? ""}`); + } + + override onError(step: string | null, _error: unknown, order: Order): void { + this.events.push(`error:${step ?? ""}:${order.price}`); + } +} + +class PricingPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new DiscountFilter()); + this.addValve(new AtLeastPredicate(), new MarkBulkFilter()); + } +} + +test("every primitive takes the type", async () => { + const tap = new SeeTap(); + const hook = new RecordHook(); + const pipeline = new PricingPipeline(); + pipeline.addTap(tap); + pipeline.useHook(hook); + const given = new Order(100, 12); + const out = await pipeline.run(given); + assert.deepEqual(out, new Order(90, 12, true)); + assert.deepEqual(given, new Order(100, 12)); // the caller's value is not changed + assert.deepEqual(tap.seen, [90]); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter", "SeeTap"]); + assert.deepEqual(hook.events, ["before:", "before:DiscountFilter", "before:MarkBulkFilter"]); + await pipeline.run(new Order(100, 2)); + assert.deepEqual(pipeline.state.skipped, ["MarkBulkFilter"]); +}); + +test("a nested pipeline takes the type", async () => { + const pipeline = new Pipeline(); + pipeline.addPipeline(new PricingPipeline()); + pipeline.addFilter(new RefuseFreeFilter()); + assert.deepEqual(await pipeline.run(new Order(50, 10)), new Order(45, 10, true)); +}); + +test("a failure is the same as with a Payload", async () => { + const hook = new RecordHook(); + const pipeline = new Pipeline(); + pipeline.addFilter(new DiscountFilter()); + pipeline.addFilter(new RefuseFreeFilter()); + pipeline.useHook(hook); + await assert.rejects(pipeline.run(new Order(0, 1)), /must cost something/); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter"]); + assert.deepEqual(hook.events.slice(-2), ["error:RefuseFreeFilter:0", "error::0"]); +}); + +test("lineage does nothing for a type without _stamp", async () => { + const pipeline = new PricingPipeline(); + pipeline.observe({ lineage: true }); + assert.deepEqual(await pipeline.run(new Order(100, 12)), new Order(90, 12, true)); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter"]); +}); + +test("lineage is kept by a type with _stamp", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new PassTracedFilter()); + pipeline.observe({ lineage: true }); + const given = new TracedOrder(100, 12); + const out = await pipeline.run(given); + assert.deepEqual(out.steps, ["PassTracedFilter"]); + assert.deepEqual(given.steps, []); +}); + +test("a Payload pipeline is still the default", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new MarkRanFilter()); + assert.deepEqual((await pipeline.run(new Payload())).toDict(), { ran: true }); +}); + +// Checked by the type check (npm run typecheck), never run: a pipeline +// carries one type. +export function refusedByTheTypeCheck(): void { + const payloads = new Pipeline(); + // @ts-expect-error A Filter of Order is not a Filter of Payload. + payloads.addFilter(new DiscountFilter()); + const orders = new Pipeline(); + // @ts-expect-error A pipeline of Order does not run a Payload. + void orders.run(new Payload()); + // @ts-expect-error A type of your own must have copy(). + void new Pipeline<{ price: number }>(); +} diff --git a/packages/codeupipe-ts/tests/pricing-example.test.ts b/packages/codeupipe-ts/tests/pricing-example.test.ts index 995372f..2997e77 100644 --- a/packages/codeupipe-ts/tests/pricing-example.test.ts +++ b/packages/codeupipe-ts/tests/pricing-example.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { AuditTap } from "../examples/pricing/audit-tap.ts"; import { PricingPipeline } from "../examples/pricing/pricing-pipeline.ts"; import { Payload } from "../src/index.ts"; @@ -9,7 +10,13 @@ test("a large order gets the discount and the bulk mark", async () => { const out = await pipeline.run(new Payload({ price: 100.0, qty: 12 })); assert.deepEqual(out.toDict(), { price: 90, qty: 12, bulk: true }); assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter", "AuditTap"]); - assert.deepEqual(pipeline.state.errors.map(([label]) => label), ["AuditTap"]); + assert.deepEqual(pipeline.state.errors, []); +}); + +test("the Tap reads the Payload and prints it", (t) => { + const log = t.mock.method(console, "log", () => {}); + assert.equal(new AuditTap().observe(new Payload({ price: 90 })), undefined); + assert.deepEqual(log.mock.calls.map((call) => call.arguments), [["audit:", { price: 90 }]]); }); test("a small order gets the discount only", async () => { diff --git a/packages/codeupipe/README.md b/packages/codeupipe/README.md index 7a8649f..9f97b61 100644 --- a/packages/codeupipe/README.md +++ b/packages/codeupipe/README.md @@ -61,7 +61,7 @@ class PricingPipeline(Pipeline): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) ``` @@ -76,6 +76,31 @@ A step is a class. State and Hooks report each step by the name of its class, for example `DiscountFilter`. A lambda is refused, because it does not have the method of a step. +## A type of your own + +`Payload` is the default: a map that any developer can read. Any object with +a `copy()` method can take its place, such as a dataclass: + +```python +@dataclasses.dataclass +class Order: + price: float + + def copy(self): + return dataclasses.replace(self) + + +class DiscountFilter: + def call(self, order): + order.price = round(order.price * 0.9, 2) + return order +``` + +The rules stay the same. Lineage is kept only if the object has +`_stamp(label)`; otherwise `observe(lineage=True)` does nothing for it. The +compiler reads Payload operations only: keep such a pipeline dynamic. See +"A type of your own" in [conformance/API.md](../../conformance/API.md). + ## What is in the package - `codeupipe`: the eight primitives. diff --git a/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py b/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py index d4b8021..603612c 100644 --- a/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py @@ -12,7 +12,7 @@ from codeupipe.core import Payload -from .rewrite_payload_ops_filter import _PayloadToDictTransformer +from .rewrite_payload_ops_filter import RewritePayloadOpsFilter, _PayloadToDictTransformer __all__ = ["LowerControlFlowFilter"] @@ -29,7 +29,8 @@ def call(self, payload: Payload) -> Payload: predicate_ast = step.predicate_ast if predicate_ast is not None: name = getattr(predicate_ast, "name", "predicate") - transformer = _PayloadToDictTransformer(step.name, name, step.ident) + transformer = _PayloadToDictTransformer( + step.name, name, step.ident, RewritePayloadOpsFilter.payload_parameter(predicate_ast, after_self=False)) predicate_ast = ast.fix_missing_locations( transformer.visit(ast.parse(ast.unparse(predicate_ast)).body[0]) ) diff --git a/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py b/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py index eb0e1e2..49d3b0c 100644 --- a/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py @@ -151,7 +151,7 @@ def _literal_source(value) -> Optional[str]: @staticmethod def _parse_predicate(step, predicate) -> Tuple[ast.FunctionDef, tuple]: - """Recover a Valve predicate object's ``test`` as a ``def`` taking ``payload``. + """Recover a Valve predicate object's ``test`` as a ``def`` of its Payload. Predicates are objects (conformance/API.md, "Valve"): the ``test`` method is lowered like a filter's ``call``, the instance's literal @@ -175,19 +175,10 @@ def refuse(suggestion: str, kind: str = "FunctionDef") -> UnsupportedFilterShape if not isinstance(fn, ast.FunctionDef) or len(fn.args.args) != 2: raise refuse("A Valve predicate's test must be a plain `def test(self, payload)`.") fn, config = ParseFilterSourceFilter._bind_instance_config(fn, predicate, step, prefix="_when_") - fn.args.args = fn.args.args[1:] # self is gone: its reads are now constants - return ParseFilterSourceFilter._rename_parameter(fn, fn.args.args[0].arg, "payload"), config - - @staticmethod - def _rename_parameter(fn: ast.FunctionDef, old: str, new: str) -> ast.FunctionDef: - """Rename the predicate's parameter so RewritePayloadOps recognises it.""" - if old == new: - return fn - fn.args.args[0].arg = new - for node in ast.walk(fn): - if isinstance(node, ast.Name) and node.id == old: - node.id = new - return fn + # self is gone: its reads are now constants. The parameter keeps the name + # that the author gave it; RewritePayloadOps follows that name. + fn.args.args = fn.args.args[1:] + return fn, config @staticmethod def _detect_dependencies(fn: ast.FunctionDef, obj, step_name: str) -> Tuple[tuple, tuple]: diff --git a/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py b/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py index 951e30b..c686714 100644 --- a/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py @@ -62,12 +62,13 @@ class _PayloadToDictTransformer(ast.NodeTransformer): or a ``copy()``) becomes a name bound to a dict. """ - def __init__(self, filter_name: str, method_name: str, fn_name: str = ""): + def __init__(self, filter_name: str, method_name: str, fn_name: str = "", parameter: str = "payload"): self.filter_name = filter_name # the step's label: for messages self.method_name = method_name self.fn_name = fn_name or filter_name # the generated function's name # source name bound to a Payload -> the dict variable it becomes - self.payloads = {"payload": "data"} + self.parameter = parameter # the name the step gives its Payload + self.payloads = {parameter: "data"} self._temps = 0 # -- which expressions are Payloads --------------------------------- @@ -164,7 +165,7 @@ def visit_Assign(self, node: ast.Assign): source = node.targets[0].id statements: list = [] value = self._lower_payload(node.value, statements) - variable = self.payloads.setdefault(source, "data" if source == "payload" else source) + variable = self.payloads.setdefault(source, "data" if source == self.parameter else source) if not (isinstance(value, ast.Name) and value.id == variable): statements.append(ast.Assign(targets=[_name(variable, ast.Store())], value=value)) return self._located(statements or [ast.Pass()], node) @@ -184,10 +185,10 @@ def visit_Attribute(self, node: ast.Attribute): and node.attr not in _LOWERED_MEMBERS): if node.attr in _OBSERVABILITY_MEMBERS: raise self._error(node, "Attribute", - f"payload.{node.attr} has no static form: exported steps work on a " + f"{node.value.id}.{node.attr} has no static form: exported steps work on a " "plain dict, which carries no trace ID or lineage.") raise self._error(node, "Attribute", - f"payload.{node.attr} is not an exportable Payload operation " + f"{node.value.id}.{node.attr} is not an exportable Payload operation " f"({', '.join(sorted(_LOWERED_MEMBERS))}).") return self.generic_visit(node) @@ -258,12 +259,15 @@ def call(self, payload: Payload) -> Payload: rewritten_steps = [] for step in pipeline_ir.steps: - call_transformer = _PayloadToDictTransformer(step.name, step.call_ast.name, step.ident) + call_transformer = _PayloadToDictTransformer(step.name, step.call_ast.name, step.ident, + RewritePayloadOpsFilter.payload_parameter(step.call_ast)) call_ast = ast.fix_missing_locations(call_transformer.visit(ast.parse(ast.unparse(step.call_ast)).body[0])) predicate_ast = step.predicate_ast if predicate_ast is not None: predicate_name = getattr(predicate_ast, "name", "predicate") - predicate_transformer = _PayloadToDictTransformer(step.name, predicate_name, step.ident) + predicate_transformer = _PayloadToDictTransformer( + step.name, predicate_name, step.ident, + RewritePayloadOpsFilter.payload_parameter(predicate_ast, after_self=False)) predicate_ast = ast.fix_missing_locations( predicate_transformer.visit(ast.parse(ast.unparse(predicate_ast)).body[0]) ) @@ -272,6 +276,15 @@ def call(self, payload: Payload) -> Payload: return payload.insert("pipeline_ir", replace(pipeline_ir, steps=rewritten_steps)) + @staticmethod + def payload_parameter(fn: ast.FunctionDef, after_self: bool = True) -> str: + """The name that a step gives its Payload: the parameter after ``self``, + or the first one for a Valve's predicate, whose ``self`` is already gone + (its reads are constants). A step that calls it ``order`` compiles as + one that calls it ``payload``.""" + at = 1 if after_self else 0 + return fn.args.args[at].arg if len(fn.args.args) > at else "payload" + @staticmethod def _drop_unused_framework_imports(imports: tuple, *trees: ast.AST) -> tuple: """Lowering removes ``Payload`` uses; drop their imports diff --git a/packages/codeupipe/codeupipe/core/pipeline.py b/packages/codeupipe/codeupipe/core/pipeline.py index c3b4d1e..091de6a 100644 --- a/packages/codeupipe/codeupipe/core/pipeline.py +++ b/packages/codeupipe/codeupipe/core/pipeline.py @@ -172,6 +172,15 @@ async def _run_fast(self, initial_payload: Payload[TInput], state: State) -> Pay return payload # type: ignore[return-value] + @staticmethod + def _stamp(payload: Any, label: str) -> None: + """Lineage: record that a step ran. Payload keeps the label. A type of + your own keeps it only if it has ``_stamp(label)``; for any other type, + lineage does nothing. The return value of ``_stamp`` is not used.""" + stamp = getattr(payload, "_stamp", None) + if stamp is not None: + stamp(label) + @staticmethod async def _invoke(fn, *args): """Call fn(*args), awaiting the result only if it is a coroutine.""" @@ -246,7 +255,7 @@ async def run_async(self, initial_payload: Payload[TInput], state: Optional[Stat if self._observe_timing: state.record_timing(name, time.monotonic() - _step_t0) if self._observe_lineage: - payload = payload._stamp(name) + self._stamp(payload, name) except Exception as e: if _step_t0 is not None and _step_name is not None and self._observe_timing: @@ -413,7 +422,7 @@ def _run_sync(self, initial_payload: Payload[TInput], state: Optional[State]) -> if timed: state.record_timing(name, time.monotonic() - _step_t0) if self._observe_lineage: - payload = payload._stamp(name) + self._stamp(payload, name) except Exception as e: if timed and _step_t0 is not None and _step_name is not None: diff --git a/pyproject.toml b/pyproject.toml index a93f0e1..ee6e44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,7 @@ [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] -# The speed check measures time, so it is not in the default run: pytest -m perf -addopts = ["--tb=short", "-m", "not perf"] +addopts = ["--tb=short"] asyncio_mode = "strict" markers = [ "unit: Unit tests", @@ -16,5 +15,4 @@ markers = [ "core: Core functionality tests", "asyncio: Async tests", "slow: Slow tests (skippable during dev with -m \"not slow\")", - "perf: The speed check against the recorded baseline (python conformance/bench.py --check)", ] diff --git a/tests/compile/fixtures/parameter_names.py b/tests/compile/fixtures/parameter_names.py new file mode 100644 index 0000000..6e06964 --- /dev/null +++ b/tests/compile/fixtures/parameter_names.py @@ -0,0 +1,51 @@ +"""Fixture: steps whose Payload parameter is not called ``payload``. The +export must follow the name the author chose.""" + +from codeupipe.core import Pipeline + + +class DiscountFilter: + def call(self, order): + return order.insert("price", round(order.get("price", 0) * 0.9, 2)) + + +class LabelFilter: + def call(self, p): + payload = "kept" # a local that happens to be called payload + same = p + same.insert("label", payload) + return same + + +class TotalFilter: + def call(self, data): + return data.insert("total", data.get("price", 0) * data.get("qty", 0)) + + +class MarkBulkFilter: + def call(self, order): + return order.insert("bulk", True) + + +class AtLeastPredicate: + def __init__(self, minimum): + self.minimum = minimum + + def test(self, order): + payload = self.minimum # a local that happens to be called payload + return order.get("qty", 0) >= payload + + +class AuditTap: + def observe(self, seen): + seen.get("price") + + +def build() -> Pipeline: + pipeline = Pipeline() + pipeline.add_filter(DiscountFilter()) + pipeline.add_filter(LabelFilter()) + pipeline.add_filter(TotalFilter()) + pipeline.add_valve(AtLeastPredicate(10), MarkBulkFilter()) + pipeline.add_tap(AuditTap()) + return pipeline diff --git a/tests/compile/test_export_parameter_name.py b/tests/compile/test_export_parameter_name.py new file mode 100644 index 0000000..77e6570 --- /dev/null +++ b/tests/compile/test_export_parameter_name.py @@ -0,0 +1,55 @@ +"""A step may call its Payload parameter what it likes (``order``, ``p``). +The export follows that name: the compiled code gives the same output as the +dynamic run, and a refusal names the parameter as the author wrote it.""" + +import subprocess +import sys + +import pytest + +from codeupipe.compile.__main__ import main +from codeupipe.core import Payload +from tests.compile.fixtures import parameter_names + +TARGET = "tests.compile.fixtures.parameter_names:build" +INPUTS = [{"price": 100.0, "qty": 12}, {"price": 19.99, "qty": 2}, {}] +MODES = [[], ["--lowering", "inline"], ["--observe"]] + + +@pytest.mark.parametrize("mode", MODES, ids=["steps", "inline", "observe"]) +def test_the_compiled_code_gives_the_same_output(tmp_path, mode): + out = tmp_path / "out" + assert main([TARGET, "--out", str(out)] + mode) == 0 + + for data in INPUTS: + script = "import sys; sys.path.insert(0, %r); import steps; print(steps.run(%r))" % (str(out / "src"), data) + done = subprocess.run([sys.executable, "-I", "-c", script], capture_output=True, text=True, cwd=tmp_path) + + assert done.returncode == 0, done.stderr + assert eval(done.stdout) == parameter_names.build().run(Payload(dict(data))).to_dict() + + +class _WritesAnAttribute: + def call(self, order): + order.price = 1 + return order + + +class _ReadsAnAttribute: + def call(self, order): + return order.insert("price", order.price) + + +@pytest.mark.parametrize("step", [_WritesAnAttribute, _ReadsAnAttribute]) +@pytest.mark.parametrize("lowering", ["steps", "inline"]) +def test_an_attribute_of_the_parameter_is_refused_by_its_own_name(step, lowering): + from codeupipe import Pipeline + from codeupipe.compile import build_export_pipeline_v2 + from codeupipe.compile.errors import UnsupportedFilterShape + + pipeline = Pipeline() + pipeline.add_filter(step()) + with pytest.raises(UnsupportedFilterShape) as refused: + build_export_pipeline_v2().run(Payload({"pipeline": pipeline, "pattern": "flat", "lowering": lowering})) + assert "order.price" in str(refused.value) + assert step.__name__ in str(refused.value) diff --git a/tests/conformance/test_bench_baseline.py b/tests/conformance/test_bench_baseline.py index 77b5633..85964f4 100644 --- a/tests/conformance/test_bench_baseline.py +++ b/tests/conformance/test_bench_baseline.py @@ -1,7 +1,9 @@ -"""The speed check: each benchmark number against its recorded baseline. +"""The comparison of benchmark numbers with a recorded baseline +(bench.py --check). No test and no CI job runs the comparison itself: a +shared machine is too noisy to judge speed. It is kept for a machine whose +resources are fixed, such as a constrained container. -A number is compared as a ratio to a reference workload timed beside it, so -the check does not depend on how fast or how busy the machine is. +A number is compared as a ratio to a reference workload timed beside it. """ import pathlib @@ -63,16 +65,31 @@ def test_a_new_number_with_no_baseline_fails(): def test_the_baseline_file_holds_a_ratio_for_every_number(): baseline = bench.load_baseline() assert baseline, "run: python conformance/bench.py --record" - for version, recorded in baseline.items(): - assert set(recorded["relative"]) == set(bench.BENCH_TARGETS), version + for key, recorded in baseline.items(): + runtime = "rust" if key == "rust" else "python" # the other keys are Python versions + want = {target for target in bench.BENCH_TARGETS if target.split(":")[0] == runtime} + assert set(recorded["relative"]) == want, key for target, row in recorded["relative"].items(): - assert row and all(ratio > 0 for ratio in row.values()), (version, target) + assert row and all(ratio > 0 for ratio in row.values()), (key, target) -@pytest.mark.perf -def test_nothing_is_slower_or_faster_than_its_baseline(): - if bench.python_key() not in bench.load_baseline(): - pytest.skip("no baseline for Python %s: python conformance/bench.py --record" % bench.python_key()) - done = subprocess.run([sys.executable, str(CONFORMANCE / "bench.py"), "--check"], - capture_output=True, text=True, timeout=600) - assert done.returncode == 0, done.stdout[-3000:] + done.stderr[-2000:] +def test_a_rust_number_is_kept_apart_from_the_python_version(): + # The speed of the Rust runtime does not depend on the Python that runs bench.py. + assert bench.baseline_key("python:run", python="3.12") == "3.12" + assert bench.baseline_key("rust:run", python="3.12") == "rust" + + +def test_the_check_compares_each_target_with_its_own_baseline(): + baseline = {"3.12": {"relative": {"python:run": {"m": 5.0}, "python:static-inline": {"m": 0.5}}}, + "3.9": {"relative": {"python:run": {"m": 9.0}}}, + "rust": {"relative": {"rust:run": {"m": 1.0}}}} + assert bench.baseline_for(baseline, ["python:run", "rust:run"], python="3.12") == { + "relative": {"python:run": {"m": 5.0}, "rust:run": {"m": 1.0}}} + + +def test_a_record_replaces_only_the_targets_that_were_measured(): + baseline = {"3.12": {"relative": {"python:run": {"m": 5.0}}, "us_per_run": {"python:run": {"m": 4.0}}}} + now = {"relative": {"rust:run": {"m": 1.0}}, "us_per_run": {"rust:run": {"m": 0.9}}} + bench.record_into(baseline, now, python="3.12") + assert baseline == {"3.12": {"relative": {"python:run": {"m": 5.0}}, "us_per_run": {"python:run": {"m": 4.0}}}, + "rust": {"relative": {"rust:run": {"m": 1.0}}, "us_per_run": {"rust:run": {"m": 0.9}}}} diff --git a/tests/conformance/test_judge_pipeline.py b/tests/conformance/test_judge_pipeline.py index d9049fe..00fa78b 100644 --- a/tests/conformance/test_judge_pipeline.py +++ b/tests/conformance/test_judge_pipeline.py @@ -61,6 +61,12 @@ def test_a_python_target_always_runs_and_a_typescript_target_needs_node_and_its_ assert DefaultTargetsFilter(lambda: False).choose(targets, needs) == ["python:run"] +def test_a_rust_target_needs_cargo(): + targets = {"python:run": {}, "rust:run": {}} + assert DefaultTargetsFilter(lambda: False, lambda: True).choose(targets, {}) == ["python:run", "rust:run"] + assert DefaultTargetsFilter(lambda: False, lambda: False).choose(targets, {}) == ["python:run"] + + def test_two_cases_with_one_id_stop_the_run(tmp_path): (tmp_path / "a.json").write_text(json.dumps([{"id": "x"}, {"id": "x"}])) with pytest.raises(SystemExit, match="duplicate case ids"): diff --git a/tests/conformance/test_rust_target.py b/tests/conformance/test_rust_target.py new file mode 100644 index 0000000..5953cf7 --- /dev/null +++ b/tests/conformance/test_rust_target.py @@ -0,0 +1,32 @@ +"""Rust must pass the same conformance cases as Python. + +The test needs cargo. On a machine with no cargo it is skipped. +""" + +import json +import pathlib +import shutil +import sys + +import pytest + +CONFORMANCE = pathlib.Path(__file__).resolve().parents[2] / "conformance" +sys.path.insert(0, str(CONFORMANCE)) + +import run as conformance_run # noqa: E402 + + + +@pytest.mark.skipif(shutil.which("cargo") is None, reason="needs cargo") +def test_rust_passes_every_case(): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target("rust:run", conformance_run.TARGETS["rust:run"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + not_passed = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] != "pass"} + assert not_passed == {}, json.dumps(not_passed, indent=2) + + +def test_cargo_is_not_needed_for_the_default_targets(monkeypatch): + monkeypatch.setattr(conformance_run, "_has_cargo", lambda: False) + assert [name for name in conformance_run.default_targets() if name.startswith("rust:")] == [] diff --git a/tests/conformance/test_typescript_target.py b/tests/conformance/test_typescript_target.py index 47c1850..fa1f590 100644 --- a/tests/conformance/test_typescript_target.py +++ b/tests/conformance/test_typescript_target.py @@ -52,4 +52,4 @@ def test_a_target_that_cannot_run_is_a_failure(monkeypatch, tmp_path): def test_node_is_not_needed_for_the_default_targets(monkeypatch): monkeypatch.setattr(conformance_run, "_node_runs_typescript", lambda: False) assert [name for name in conformance_run.default_targets() if name.startswith("typescript:")] == [] - assert len(conformance_run.default_targets()) == 5 # every Python target + assert len([name for name in conformance_run.default_targets() if name.startswith("python:")]) == 5 diff --git a/tests/core/test_custom_payload.py b/tests/core/test_custom_payload.py new file mode 100644 index 0000000..01bccdb --- /dev/null +++ b/tests/core/test_custom_payload.py @@ -0,0 +1,151 @@ +"""A pipeline over a type of your own. + +Payload is the default: a map that any developer can read. A type of your +own can take its place. The Pipeline asks one thing of it, ``copy()``, for +the copy that a run works on. Lineage is kept only by a type that has +``_stamp(label)``; for any other type, turning lineage on does nothing. +""" + +import asyncio +import dataclasses +from typing import Optional + +import pytest + +from codeupipe import Hook, Pipeline + + +@dataclasses.dataclass +class Order: + price: float + qty: int + bulk: bool = False + + def copy(self): + return dataclasses.replace(self) + + +@dataclasses.dataclass +class TracedOrder(Order): + steps: list = dataclasses.field(default_factory=list) + + def copy(self): + return dataclasses.replace(self, steps=list(self.steps)) + + def _stamp(self, label): + self.steps.append(label) + + +class DiscountFilter: + def call(self, order): + order.price = round(order.price * 0.9, 2) + return order + + +class MarkBulkFilter: + def call(self, order): + order.bulk = True + return order + + +class RefuseFreeFilter: + def call(self, order): + if order.price <= 0: + raise ValueError("an order must cost something") + return order + + +class AtLeastPredicate: + def test(self, order): + return order.qty >= 10 + + +class SeeTap: + def __init__(self): + self.seen = [] + + def observe(self, order): + self.seen.append(order.price) + + +class RecordHook(Hook): + def __init__(self): + super().__init__() + self.events = [] + + def before(self, step: Optional[str], payload) -> None: + self.events.append(f"before:{step or ''}") + + def on_error(self, step: Optional[str], error: Exception, payload) -> None: + self.events.append(f"error:{step or ''}:{payload.price}") + + +class PricingPipeline(Pipeline): + def __init__(self): + super().__init__() + self.add_filter(DiscountFilter()) + self.add_valve(AtLeastPredicate(), MarkBulkFilter()) + + +def _pricing(): + return PricingPipeline() + + +def test_every_primitive_takes_the_type(): + tap, hook = SeeTap(), RecordHook() + pipeline = _pricing() + pipeline.add_tap(tap) + pipeline.use_hook(hook) + given = Order(100.0, 12) + out = pipeline.run(given) + assert out == Order(90.0, 12, True) + assert given == Order(100.0, 12) # the caller's value is not changed + assert tap.seen == [90.0] + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter", "SeeTap"] + assert hook.events == ["before:", "before:DiscountFilter", "before:MarkBulkFilter"] + pipeline.run(Order(100.0, 2)) + assert pipeline.state.skipped == ["MarkBulkFilter"] + + +def test_the_fast_path_and_run_async_take_the_type(): + assert _pricing().run(Order(100.0, 12)) == Order(90.0, 12, True) + assert asyncio.run(_pricing().run_async(Order(100.0, 12))) == Order(90.0, 12, True) + + +def test_a_nested_pipeline_takes_the_type(): + pipeline = Pipeline() + pipeline.add_pipeline(PricingPipeline()) + pipeline.add_filter(RefuseFreeFilter()) + assert pipeline.run(Order(50.0, 10)) == Order(45.0, 10, True) + + +def test_a_failure_is_the_same_as_with_a_payload(): + hook = RecordHook() + pipeline = Pipeline() + pipeline.add_filter(DiscountFilter()) + pipeline.add_filter(RefuseFreeFilter()) + pipeline.use_hook(hook) + with pytest.raises(ValueError, match="must cost something"): + pipeline.run(Order(0.0, 1)) + assert pipeline.state.executed == ["DiscountFilter"] + assert hook.events[-2:] == ["error:RefuseFreeFilter:0.0", "error::0.0"] + + +@pytest.mark.parametrize("run", ["sync", "async"]) +def test_lineage_does_nothing_for_a_type_without_stamp(run): + pipeline = _pricing() + pipeline.observe(lineage=True) + given = Order(100.0, 12) + out = pipeline.run(given) if run == "sync" else asyncio.run(pipeline.run_async(given)) + assert out == Order(90.0, 12, True) + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter"] + + +@pytest.mark.parametrize("run", ["sync", "async"]) +def test_lineage_is_kept_by_a_type_with_stamp(run): + pipeline = _pricing() + pipeline.observe(lineage=True) + given = TracedOrder(100.0, 12) + out = pipeline.run(given) if run == "sync" else asyncio.run(pipeline.run_async(given)) + assert out.steps == ["DiscountFilter", "MarkBulkFilter"] + assert given.steps == [] diff --git a/tests/examples/test_pricing.py b/tests/examples/test_pricing.py index 94ede71..dc8a72b 100644 --- a/tests/examples/test_pricing.py +++ b/tests/examples/test_pricing.py @@ -1,6 +1,6 @@ """The README example. Each primitive is in its own file, and the pipeline -file puts them together. It runs, a failing Tap does not halt it, and its -export gives the same answer with no framework import.""" +file puts them together. It runs, and its export gives the same answer +with no framework import.""" import importlib.util import pathlib @@ -10,6 +10,7 @@ from codeupipe import Payload from examples.pricing.at_least_predicate import AtLeastPredicate +from examples.pricing.audit_tap import AuditTap from examples.pricing.discount_filter import DiscountFilter from examples.pricing.mark_bulk_filter import MarkBulkFilter from examples.pricing.pricing_pipeline import PricingPipeline @@ -50,7 +51,8 @@ def test_dynamic_run(): assert pipeline.labels == ["DiscountFilter", "MarkBulkFilter", "AuditTap"] - assert [name for name, _ in pipeline.state.errors] == ["AuditTap"] + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + assert pipeline.state.errors == [] def test_valve_skips_small_orders(): @@ -60,12 +62,17 @@ def test_valve_skips_small_orders(): assert pipeline.state.skipped == ["MarkBulkFilter"] -def test_a_hook_sees_each_step(capsys): +def test_the_tap_reads_the_payload(capsys): + assert AuditTap().observe(Payload({"price": 90.0})) is None + assert capsys.readouterr().out == "audit: {'price': 90.0}\n" + + +def test_a_hook_sees_each_step_but_not_the_tap(capsys): pipeline = PricingPipeline() pipeline.use_hook(PrintStepsHook()) pipeline.run(Payload({"price": 100.0, "qty": 12})) lines = capsys.readouterr().out.splitlines() - assert lines == [" DiscountFilter", " MarkBulkFilter"] + assert lines == [" DiscountFilter", " MarkBulkFilter", "audit: {'price': 90.0, 'qty': 12, 'bulk': True}"] def test_the_example_runs_as_a_module(): diff --git a/tests/lint/invariants/test_repo_is_clean.py b/tests/lint/invariants/test_repo_is_clean.py index cc7de64..b610704 100644 --- a/tests/lint/invariants/test_repo_is_clean.py +++ b/tests/lint/invariants/test_repo_is_clean.py @@ -8,7 +8,7 @@ from ._support import REPO # The places with a `cup: allow-plain` comment. -PLAIN_BY_DESIGN = 60 +PLAIN_BY_DESIGN = 63 CHECKED = ["packages/codeupipe/codeupipe", "conformance", "scorecard", "packages/codeupipe-lint/codeupipe_lint", "tests", "examples"]