diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b957ad..de79c58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,10 +34,96 @@ jobs: - name: Lint, anonymous steps run: python -m codeupipe_lint.anonymous_steps - name: Lint, invariants - run: python -m codeupipe_lint.invariants + run: python -m codeupipe_lint.invariants --strict - name: Conformance run: python conformance/run.py + typescript: + name: TypeScript packages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: actions/setup-node@v5 + with: + node-version: "24" + - name: Install + run: npm ci + - name: Build + run: npm run build + - name: Type check + run: npm run typecheck + - name: Tests + run: npm test + - name: Example + run: node packages/codeupipe-ts/examples/pricing/main.ts + - name: The linter on the TypeScript code of this repository + run: npm run lint -w codeupipe-lint + - name: Install the Python package, for the judge + # conformance/run.py is a codeupipe pipeline (conformance/judging/). + run: python -m pip install -e packages/codeupipe + - name: Conformance + run: python conformance/run.py typescript:run typescript:static + - name: The packages as npm makes them + run: npm pack --workspaces --dry-run + + rust: + name: Rust packages + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + # The runner image has rustup and a stable toolchain. Nothing is + # downloaded from crates.io: the crates have no dependencies. + - name: Format + run: cargo fmt --all --check + - name: Clippy + run: cargo clippy --offline --all-targets -- -D warnings + - name: Tests + run: cargo test --offline + - name: Examples + run: | + cargo run --offline --example pricing -p codeupipe + cargo run --offline --example typed_pricing -p codeupipe + - name: Install the Python package, for the judge + # conformance/run.py is a codeupipe pipeline (conformance/judging/). + run: python -m pip install -e packages/codeupipe + - name: Conformance + run: python conformance/run.py rust:run + - name: The oldest Rust that the crate supports (rust-version) + run: | + rustup toolchain install 1.75 --profile minimal + cargo +1.75 test --offline -p codeupipe + + rust-lint: + name: Rust linter + runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/codeupipe-lint-rs + steps: + - uses: actions/checkout@v5 + # A workspace of its own: it downloads syn and serde_json from + # crates.io, at the versions in its Cargo.lock. + - name: Format + run: cargo fmt --check + - name: Clippy + run: cargo clippy --locked --all-targets -- -D warnings + - name: Tests + run: cargo test --locked + - name: The linter on the Rust code of this repository + run: >- + cargo run --locked -q -- --strict src tests ../codeupipe-rs/src + ../codeupipe-rs/examples ../codeupipe-rs/tests ../../conformance/runners/rust/src + - name: The oldest Rust that the crate supports (rust-version) + run: | + rustup toolchain install 1.75 --profile minimal + cargo +1.75 test --locked + export: name: Export against dynamic run runs-on: ubuntu-latest @@ -76,11 +162,12 @@ jobs: /tmp/installed/bin/python -m codeupipe_lint.invariants --rules /tmp/installed/bin/python -m codeupipe_lint.anonymous_steps --rules - speed: - name: Speed check (does not block) + benchmark: + name: Benchmark (numbers only, never fails the run) runs-on: ubuntu-latest - # The baseline was recorded on a different machine. This job shows how - # far a CI machine is from it; it does not fail the run until that is known. + # A shared CI machine is too noisy to judge speed, so nothing here is + # compared with a baseline. The job measures and keeps the numbers: in + # the job summary and as an artifact. continue-on-error: true steps: - uses: actions/checkout@v5 @@ -91,5 +178,13 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e packages/codeupipe -e packages/codeupipe-lint -r requirements-dev.txt - - name: Compare with the recorded baseline - run: python conformance/bench.py --check + - name: Measure + run: | + set -o pipefail + python conformance/bench.py | tee -a "$GITHUB_STEP_SUMMARY" + - name: Keep the numbers + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark + path: conformance/results/bench-latest.* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5aa0a6..a1cd1f7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ name: Release -# Runs when a version tag is pushed: v1.0.0, v1.0.1, ... +# Runs when a version tag is pushed: v1.0.0, v1.0.1, and a beta such as v2.0.0b1. +# PyPI keeps a beta apart: pip installs it only when it is asked for. # Publishes the two packages to PyPI with trusted publishing: PyPI accepts # the upload because it comes from this workflow in this repository. No # token is stored anywhere. diff --git a/.gitignore b/.gitignore index f839347..003e60c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ packages/*/build/ conformance/results/ .venv-*/ scorecard/results/ +node_modules/ + +/target/ +/packages/codeupipe-lint-rs/target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c989904 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "codeupipe" +version = "0.1.0" + +[[package]] +name = "cup-conformance-rust" +version = "0.1.0" +dependencies = [ + "codeupipe", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8d7a3de --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +# The Cargo workspace of the Rust packages and the Rust conformance runner. +# It is not a package. +[workspace] +resolver = "2" +members = ["packages/codeupipe-rs", "conformance/runners/rust"] diff --git a/README.md b/README.md index 89e3b1e..dbf14d7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # codeupipe -codeupipe is a Python library for writing a program as a pipeline of small -steps. +codeupipe is a library for writing a program as a pipeline of small steps. +It is for Python and for TypeScript. There are eight primitives. Each one does one thing. @@ -28,23 +28,40 @@ If you know Unix, these are close to things you know. | Hook | A git hook: code that runs before and after a step | | State | `$?` and `PIPESTATUS` | -This repository has two packages. +This repository has seven packages. | Package | Contents | Reference | |---|---|---| -| `codeupipe` | The eight primitives. An export command that writes a pipeline as plain Python with no import of codeupipe. A config file reader. No dependencies | [conformance/API.md](conformance/API.md), [docs/static-export.md](docs/static-export.md), [docs/config.md](docs/config.md) | +| `codeupipe` | The eight primitives. A compile command that writes a pipeline as plain Python with no import of codeupipe. A config file reader. No dependencies | [conformance/API.md](conformance/API.md), [docs/compile.md](docs/compile.md), [docs/config.md](docs/config.md) | | `codeupipe-lint` | Two linters for pipeline code. Depends on `codeupipe` only | [packages/codeupipe-lint/README.md](packages/codeupipe-lint/README.md) | +| `codeupipe` for TypeScript | The eight primitives. No dependencies | [packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) | +| `codeupipe-lint` for TypeScript | A linter for pipeline code in TypeScript | [packages/codeupipe-lint-ts/README.md](packages/codeupipe-lint-ts/README.md) | +| `codeupipe-compile` for TypeScript | Writes a pipeline as plain TypeScript with no import of codeupipe | [packages/codeupipe-compile-ts/README.md](packages/codeupipe-compile-ts/README.md) | +| `codeupipe` for Rust | The primitives, 0.1.0. No dependencies. No compiler yet | [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) | +| `codeupipe-lint` for Rust | A linter for pipeline code in Rust, for the rules that the compiler does not check | [packages/codeupipe-lint-rs/README.md](packages/codeupipe-lint-rs/README.md) | ## Install -Version 1.0.0 is on PyPI. Python 3.9 or later. No dependencies. +Python 3.9 or later. No dependencies. + +| Version | State | Install | +|---|---|---| +| 1.0.0 | Stable. On PyPI | `pip install codeupipe` | +| 2.0.0b1 | Beta. The `main` branch of this repository. Not on PyPI yet | From this repository, see below | + +This README describes version 2, the beta. A beta can change in a way that +is not compatible, from one beta to the next. `pip install codeupipe` does +not install a beta: pip installs one only when you ask for it, with +`pip install --pre codeupipe` or with the exact version. + +Until the beta is on PyPI, install it from this repository: ```bash -pip install codeupipe +pip install "codeupipe @ git+https://github.com/orchestrate-solutions/codeupipe@main#subdirectory=packages/codeupipe" +pip install "codeupipe-lint @ git+https://github.com/orchestrate-solutions/codeupipe@main#subdirectory=packages/codeupipe-lint" ``` -`codeupipe-lint` 1.0.0 is not on PyPI yet. Until it is, install it from this -repository: +`codeupipe-lint` 1.0.0 is not on PyPI. Install it from the tag `v1.0.0`: ```bash pip install "codeupipe-lint @ git+https://github.com/orchestrate-solutions/codeupipe@v1.0.0#subdirectory=packages/codeupipe-lint" @@ -53,6 +70,29 @@ pip install "codeupipe-lint @ git+https://github.com/orchestrate-solutions/codeu Version 1.0.0 is not compatible with `codeupipe` 0.10.0. It is a new library with the same name. +### Changes from 1.0.0 to 2.0.0b1 + +These changes are not compatible with 1.0.0. + +- **A step is reported by a label, the name of its class.** State and Hooks + give `DiscountFilter`, where 1.0.0 gave the full path + `examples.pricing.discount_filter.DiscountFilter`. +- **Removed:** `AnonymousStep`, `Pipeline.identities` (use + `Pipeline.labels`), `Pipeline.step_sites`, and lint rule CUP-A006. +- **`codeupipe.lower` is now `codeupipe.compile`.** The command is + `python -m codeupipe.compile`. + +The TypeScript packages are new. They are version `0.1.0`. A version below +1.0 is not stable yet, and a new version can change in a way that is not +compatible. They are not on npm yet: +[packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) says how to +install them from this repository. + +The Rust crate is new too, version `0.1.0` on the same terms. It is not on +crates.io yet: +[packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) says how to +use it from this repository. + ## Example [examples/pricing/](examples/pricing/) has one class in each file. The @@ -108,7 +148,7 @@ class PricingPipeline(Pipeline): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) ``` @@ -123,10 +163,9 @@ A Valve is made in the pipeline file: `add_valve(predicate, filter)`. It joins a Predicate to a Filter. The Filter runs only when `predicate.test(payload)` is true. There is no Valve file. -A step must be a class in a module. A lambda, or a class in the script that -you run, is refused. The reason is step identity: a step is identified by its -type, for example `examples.pricing.discount_filter.DiscountFilter`. State, -Hooks and the export use that identity. +A step is a class. State and Hooks report each step by the name of its +class, for example `DiscountFilter`. A lambda is refused, because it does not +have the method of a step. ```bash python -m examples.pricing @@ -151,32 +190,46 @@ An editable install needs pip 21.3 or later. | Command | What it does | |---|---| | `python -m pytest tests -q` | Runs the tests | -| `python conformance/run.py` | Runs the 27 conformance cases against the dynamic run and the three exports | +| `python conformance/run.py` | Runs the 30 conformance cases against the Python dynamic run and the three compiled forms, and against the TypeScript targets that the machine can run | | `python -m codeupipe_lint.anonymous_steps` | Runs the anonymous-step linter. Paths come from `cup.config.json` | | `python -m codeupipe_lint.invariants` | Runs the invariant linter. Paths come from `cup.config.json` | -| `python -m codeupipe.lower examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing` | Exports the example to `build/pricing/src/steps/` | -| `python conformance/bench.py --check` | Compares benchmark numbers with the recorded baseline. Fails outside 25% | -| `python -m scorecard.differential` | Compares the export with the dynamic run on random pipelines | +| `python -m codeupipe.compile examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing` | Compiles the example to `build/pricing/src/steps/` | +| `python conformance/bench.py` | Measures every target on the benchmark scenarios. It reports numbers and judges nothing | +| `python -m scorecard.differential` | Compares the compiled code with the dynamic run on random pipelines | ## Layout | Path | Contents | |---|---| -| `packages/codeupipe/` | The `codeupipe` package: `core/`, `lower/` (the export), `config.py` | +| `packages/codeupipe/` | The `codeupipe` package: `core/`, `compile/` (the compiler), `config.py` | | `packages/codeupipe-lint/` | The `codeupipe-lint` package: `anonymous_steps/`, `invariants/` | -| `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runner, the benchmark | -| `tests/` | The tests for the two packages | +| `packages/codeupipe-ts/` | The TypeScript runtime, with its tests and its example | +| `packages/codeupipe-lint-ts/` | The TypeScript linter, with its tests | +| `packages/codeupipe-compile-ts/` | The TypeScript compiler, with its tests | +| `packages/codeupipe-rs/` | The Rust runtime, with its tests and its examples | +| `packages/codeupipe-lint-rs/` | The Rust linter, with its tests. A Cargo workspace of its own | +| `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runners, the benchmark | +| `tests/` | The tests for the two Python packages | | `examples/pricing/` | The example, with its test in `tests/examples/` | -| `scorecard/` | The comparison of the export with the dynamic run on random pipelines | -| `docs/` | Reference for the export and the config file | +| `scorecard/` | The comparison of the compiled code with the dynamic run on random pipelines | +| `docs/` | Reference for the compiler and the config file | -The root `pyproject.toml` is not a package. It holds the test settings. +The root `pyproject.toml` is not a package. It holds the test settings. The +root `package.json` is not a package: it makes the three TypeScript packages +one npm workspace. The root `Cargo.toml` is not a package either: it makes +the Rust runtime and its conformance runner one Cargo workspace, which +builds with no download. The Rust linter needs `syn`, so it is a workspace +of its own. ## Other languages -This repository is Python only. The contract and the conformance cases do -not depend on Python. A runtime in another language must implement the -contract and add a conformance runner. +The contract and the conformance cases do not depend on a language. Python +is the first runtime and TypeScript is the second +([packages/codeupipe-ts/](packages/codeupipe-ts/)), with its own linter and +its own compiler. Rust is the third +([packages/codeupipe-rs/](packages/codeupipe-rs/)), with its own linter; its plan is +[docs/rust-plan.md](docs/rust-plan.md). A runtime in another language must implement +the contract and add a conformance runner. ## License diff --git a/conformance/API.md b/conformance/API.md index 3caaa39..a822452 100644 --- a/conformance/API.md +++ b/conformance/API.md @@ -14,9 +14,10 @@ parameters, their defaults, and how they fail. | State | records what happened | | Pipeline | runs the steps in order | -Python implements it. A runtime in another language implements the same -contract, with identifier casing in that language's style, and is judged by -the same conformance cases. +Python, TypeScript and Rust implement it. Each runtime has the same contract, +with identifier casing in the style of its language, and the same +conformance cases judge it. The tables below show the Python names. The +TypeScript and Rust names and differences are in "Runtimes". ## Principles @@ -70,6 +71,24 @@ Rules: - A Tap, a Hook or a Predicate that wants its own changed version, or that keeps the Payload for later, takes a `copy()`. +### A type of your own + +The Payload is the default, and the form that this contract, the +conformance cases and the compilers use: a map that any developer can read, +which shows that anything can go there. A pipeline can carry a type of your +own instead, such as the class or struct that your steps would use anyway. +Its Filters, Predicates, Taps and Hooks then take that type. + +- **The Pipeline asks one thing of it: `copy()`**, for the copy that `run` + works on. Every rule above stays the same. +- **Lineage is kept only by a type that has `_stamp(label)`** (`stamp` in + Rust). For any other type, `observe(lineage=True)` does nothing: the run + goes on, and `state.executed` still lists the steps. +- **The compilers read Payload operations only.** A step that uses a type of + your own stays dynamic; the TypeScript compiler refuses it. + +How each runtime spells it is in "Runtimes". + ## Filter `call(payload) -> payload`. A failure raises. @@ -78,7 +97,7 @@ Rules: `test(payload) -> bool`. -- A Predicate is an object. It is identified by its class like a step, and +- A Predicate is an object. Its class gives it its name, and any setting (a threshold, a flag) is on the instance: `AtLeastPredicate("qty", 10)`. Functions, lambdas and closures are not Predicates. @@ -94,7 +113,7 @@ A Valve is added with one call, the condition first: - When `predicate.test(payload)` is true, the Filter runs. When it is false, the Payload passes through unchanged and the step is recorded in `state.skipped`, not `state.executed`. -- The step is identified by its Filter (see "Step identity"). +- The step has the label of its Filter (see "Step labels"). - `add_valve` is the one way to add a Valve. There is no Valve class for a user to make, and `add_filter` does not accept one. @@ -102,7 +121,7 @@ A Valve is added with one call, the condition first: `observe(payload)`, no return value. A Tap reads the Payload and does not write to it. **A Tap failure never halts the pipeline:** the error is -recorded in `state.errors` under the Tap's identity, the Tap still counts as +recorded in `state.errors` under the Tap's label, the Tap still counts as executed, and hooks are not notified. ## Hook — sees the lifecycle @@ -110,7 +129,7 @@ executed, and hooks are not notified. `before(step, payload)`, `after(step, payload)`, `on_error(step, error, payload)`. A Hook reads the Payload and does not write to it. -- `step` is the **step's identity** (see "Step identity"). Pipeline-level +- `step` is the **step's label** (see "Step labels"). Pipeline-level events pass absent (`None`). - `error` is the error itself (never just its message). - Order on success: `before(pipeline)`, then for each filter or Valve @@ -119,12 +138,23 @@ A Hook reads the Payload and does not write to it. - Order on a filter failure: `… before(step)`, `on_error(step)`, `on_error(pipeline)`. No `after` events follow. -## State — what happened in the last run +## State — what happened in one run -`executed` (identities, in order), `skipped` (identities), `errors` -(identity + error), `timings` (identity → seconds, when timing is on). A run -starts with a new State. **A failed run keeps the steps that completed -before the failure.** +`executed` (labels, in order), `skipped` (labels), `errors` +(label + error), `timings` (label → seconds, when timing is on). **A failed +run keeps the steps that completed before the failure.** + +**A State belongs to one run, not to the Pipeline.** + +- Each run writes into a State of its own. Two runs of one Pipeline that + are in progress at the same time never write into the same State. +- The caller can give the State: `run(p, state)`. The run writes into that + State. This is how a caller with more than one run at a time gets the + State of each run. The caller gives a new State for each run. +- With no State from the caller, the run makes a new one. +- `state` on the Pipeline is the State of the run that started last, with + a State from the caller or not. It is for a caller that has one run at a + time. ## Pipeline @@ -136,70 +166,61 @@ before the failure.** | add a tap | `add_tap(t)` | | nest a pipeline | `add_pipeline(p)` | | add a hook | `use_hook(h)` | -| run (sync) | `run(p)` | -| run (async, opt-in) | `run_async(p)` | -| last run | `state` | +| run (sync) | `run(p)`, `run(p, state)` | +| run (async, opt-in) | `run_async(p)`, `run_async(p, state)` | +| the State of the run that started last | `state` | | timing and lineage | `observe(timing=True, lineage=False)` | -| each step's identity, in order | `identities` | -| where each step was added (`file:line`) | `step_sites` | +| each step's label, in order | `labels` | -- **Steps have no names.** State and Hooks report each step's identity. +- **Steps are not named by hand.** State and Hooks report each step's label. - **`run` is synchronous** and never starts an event loop. If it meets an async step it raises `AsyncStepInSyncRun` rather than silently paying for a loop. -- A nested pipeline is one step: its identity appears once in the parent's +- A nested pipeline is one step: its label appears once in the parent's State. - **Timing and lineage are off by default.** -## Step identity: the filter, not a name - -A step's identity is a pointer to the code that runs it, derived -automatically: `add_filter(f)`, not `add_filter(f, name)`. State, Hooks and -exported code all report this identity. - -Why: in a codebase maintained by agents and reviewed by humans, an identity -taken from the code cannot drift from it. A hand-written name is an -unchecked label that can describe code which no longer matches it; a pointer -to the filter is always true, more specific, and needs no one to maintain -it. - -### The identity - -- **The filter's fully qualified symbol** (`app.filters.Validate`), plus an - **occurrence number only when the same identity appears more than once** - in one pipeline: the second is `...Validate#2`. Stable across unrelated - edits. -- **Extra detail, not identity**: the source location of the `add_*` call - (`file:line`), for agents and reviewers. -- **Valves**: the Filter's identity; the predicate's symbol is extra detail. -- **Pipelines used as a step are named types**: `class UserOnboarding(Pipeline)`, - identified by that type's symbol exactly as a filter is. A plain, unnamed - `Pipeline` can run at the top level but cannot be nested. -- Renames and moves change the identity. That is honest (it is different - code); cross-version comparisons follow renames from version control. -- A filter defined in the entry script (`__main__.X`) is refused like an - anonymous one: its identity changes with how the program is started, and - tests cannot import it. - -### Anonymous filters are unsupported - -Lambdas, closures, classes defined inside a function, inline functions: as -filters, taps, valve predicates or nested pipelines. They are not -independently testable and have no stable identity. Enforced in two layers: - -1. **The runtime refuses them** at `add_*` time (`AnonymousStep`), with a - message naming the fix. -2. **The linter flags them** at authoring time: - `python -m codeupipe_lint.anonymous_steps `. One line per - anonymous step, `path:line:col CUP-A00N slug message`; exit 1 when it - finds any. An intentional one (a test of the refusal) is suppressed with - a `cup: allow-anonymous` comment. Rules and limits: - [packages/codeupipe-lint/README.md](../packages/codeupipe-lint/README.md). +## Step labels + +A step is reported by a label. The label is the name of the step's class: +`add_filter(ValidateFilter())` gives the label `ValidateFilter`. State and +Hooks report this label. A step is not given a name by hand: +`add_filter(f)`, not `add_filter(f, name)`. + +- **A second use gets a number.** When the same label is in one pipeline + more than once, the second is `ValidateFilter#2`, the third + `ValidateFilter#3`. +- **A Valve** has the label of its Filter. +- **A nested pipeline** has the label of its class. Make it a named type + (`class UserOnboarding(Pipeline)`), so that the label says what it is. A + plain `Pipeline` that is nested has the label `Pipeline`. +- A label is for people and for records. It does not say where the class is + defined, and two classes with the same name in different modules have the + same label. + +### What is refused when a step is added + +A step must have the method of its role. `add_filter` needs `call(payload)`, +`add_tap` needs `observe(payload)`, `add_valve` needs a predicate with +`test(payload)` and a Filter with `call(payload)`, and `add_pipeline` needs a +Pipeline. A value without that method (a lambda, a plain function) is +refused with a `TypeError` that names the method. + +### What the linter flags + +A step that is a lambda, a function, a class defined inside a function, or a +plain nested `Pipeline` cannot be imported and tested alone, and its label +says little. The linter flags these when the code is written: +`python -m codeupipe_lint.anonymous_steps `. One line per finding, +`path:line:col CUP-A00N slug message`; exit 1 when it finds any. An +intentional one is suppressed with a `cup: allow-anonymous` comment. Rules +and limits: +[packages/codeupipe-lint/README.md](../packages/codeupipe-lint/README.md). **Conformance.** The shared cases name their steps; the runner builds every -step, so it maps each reported identity back to the case's step name. An -identity that is wrong, or differs between the dynamic run and an export, -does not map and fails the comparison. +step, so it maps each reported label back to the case's step name. A label +that is wrong, or differs between the dynamic run and compiled code, does not +map and fails the comparison. ## Not in the contract yet @@ -211,3 +232,50 @@ does not map and fails the comparison. it changes timing for callers who wait on a Tap's side effect. - **Other runtimes.** A runtime in another language is written to this contract and added with its conformance runner. + +## Runtimes + +| | Python | TypeScript | Rust | +|---|---|---|---| +| Package | `packages/codeupipe` | `packages/codeupipe-ts` | `packages/codeupipe-rs` (crate `codeupipe`) | +| Names | snake_case | camelCase: `addFilter`, `addValve`, `toDict`, `withTrace`, `traceId`, `onError` | snake_case, as Python | +| Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | `run` is sync, and `run_with_state` for the caller's State. No async run yet | +| Absent | `None` | `null` | `None` from `get`; `Value::Null` is a JSON null | +| Label of a step | `type(step).__name__` | `step.constructor.name` | The name of the step's type, from `std::any::type_name` | +| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | None yet | +| Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | None yet. The compiler refuses several of the shapes that the linters find | +| A type of your own | Any object with `copy()` | `class Order implements PipelineData`, then `new Pipeline()`, `Filter` | `impl PipelineData for Order {}`, then `Pipeline`, `Filter` | + +Notes on TypeScript: + +- **Principle 3.** JavaScript always has an event loop, so there is no + second entry point. Inside `run`, a step that returns a value and not a + Promise is not awaited. +- **A function is not a step.** Every JavaScript function has a `call` + method, so a step must be an object. +- **Labels in minified code.** A minifier changes class names. A label is + then the changed name, unless the build keeps class names. A class with no + name has the label `anonymous`. +- **State and concurrent runs.** Two runs of one pipeline at the same time + each write to their own State. `state` is the State of the run that + started last. + +Notes on Rust: + +- **Refused when added, at compile time.** A closure is not a step, the + arguments of `add_valve` are in one order only, and a Tap, a Hook or a + Predicate gets `&Payload`, so it cannot write. Each is a type error. +- **A Filter changes the Payload in place.** `call(&self, &mut Payload)` + returns `Result<(), StepError>`. A failure is an `Err`; a panic is not a + failure of the step and is not caught. +- **Values.** A value is a `Value`: null, a bool, a number (`f64`), a string, + a list or a map that keeps the order of its keys, as JSON has. +- **A State for each run.** `run(&self, …)` takes the pipeline by shared + reference, so a step can hold the pipeline that it is in (an `Rc`) and run + it again. The run with the caller's State is + `run_with_state(&input, &mut state)`. `state()` gives an `Rc`: the + State of the run that started last, once it has ended, as a snapshot.md](../docs/rust-plan.md) has the + plan. +- **A type of your own** is a type parameter of each primitive, with + `Payload` as its default, so a step for one type in a pipeline of another + is a compile error. TypeScript does the same with its type check. diff --git a/conformance/BENCH.md b/conformance/BENCH.md index c1cdaad..7d3dd2e 100644 --- a/conformance/BENCH.md +++ b/conformance/BENCH.md @@ -5,24 +5,27 @@ A snapshot of `python conformance/bench.py`. Scenarios: Every target's output is checked against the others before its time counts. Apple Silicon, one run. -These numbers are a snapshot for reading. The numbers that the speed check -compares are in `bench/baseline.json` (`python conformance/bench.py --check`, -[README.md](README.md)). +These numbers are a snapshot for reading. Nothing fails because of them: CI +measures again on every run and keeps the numbers, and does not compare them +([README.md](README.md)). `python:run` is the dynamic run. The three `static-*` targets are the -exports: `static-observe` records State and calls Hooks as `run` does; the -two fast exports do neither, and refuse the `hook` scenario rather than +compiles: `static-observe` records State and calls Hooks as `run` does; the +two fast modes do neither, and refuse the `hook` scenario rather than silently dropping the Hook. +`rust:run` is the dynamic run of the Rust crate, built with `--release`. It +records State and calls Hooks as `python:run` does. + µs per run (best of 5 × 20,000, after 20,000 warmup runs); lower is better. -| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | -|---|---|---|---|---|---| -| `micro-pass` | 10 | 4.92 | 0.838 | 0.580 | 1.95 | -| `valve-gated` | 10 | 5.61 | 1.15 | 0.900 | 2.30 | -| `taps` | 13 | 5.21 | 0.887 | 0.629 | 2.07 | -| `hook` | 10 | 5.96 | — | — | 3.03 | -| `heavy-allocation` | 50 | 17.14 | 2.80 | 1.49 | 7.22 | +| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | rust:run | +|---|---|---|---|---|---|---| +| `micro-pass` | 10 | 4.94 | 0.824 | 0.606 | 1.92 | 0.246 | +| `valve-gated` | 10 | 5.81 | 1.11 | 0.801 | 2.25 | 0.301 | +| `taps` | 13 | 5.34 | 0.899 | 0.602 | 2.03 | 0.256 | +| `hook` | 10 | 5.88 | — | — | 2.97 | 0.247 | +| `heavy-allocation` | 50 | 16.05 | 2.84 | 1.49 | 7.01 | 4.24 | Not measured (the target refuses the scenario): - python:static-steps hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast static export. Export with observe=True (top-level Hooks only), or remove them. diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md index 7be1505..59cb9f5 100644 --- a/conformance/MATRIX.md +++ b/conformance/MATRIX.md @@ -5,47 +5,57 @@ live copy is `results/latest.md` (not committed). The command exits non-zero if any target has a ❌. **Every target passes every case, and nothing is unsupported.** The targets -are the dynamic run, the async run, and the three static exports. +are the Python dynamic run, async run and three compiled forms, the +TypeScript run and its compiled form, and the Rust run. -The two fast exports (`static-steps`, `static-inline`) expose no State or -Hooks, so they pass those cases as ✅\*. `static-observe` (export with -`observe=True`) records both and passes every case outright. See -[docs/static-export.md](../docs/static-export.md). +The fast modes (`python:static-steps`, `python:static-inline`, +`typescript:static`) expose no State or Hooks, so they pass those cases as +✅\*. `static-observe` (compile with +`observe=True`) records both and passes every case outright, but for one: +the case that asks for `state` on the Pipeline object. A compiled form is a +function and has no such object. See +[docs/compile.md](../docs/compile.md). -| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | -|---|---|---|---|---|---| -| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | -| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | +| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | rust:run | +|---|---|---|---|---|---|---|---|---| +| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/the-caller-gives-the-state-of-a-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/a-run-that-starts-inside-another-run-has-its-own-state` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last` | ✅ | ✅ | ✅* | ✅* | ✅* | ✅ | ✅* | ✅ | +| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -- **python:run**: 27 pass, 0 pass*, 0 fail, 0 unsupported -- **python:run_async**: 27 pass, 0 pass*, 0 fail, 0 unsupported -- **python:static-steps**: 11 pass, 16 pass*, 0 fail, 0 unsupported -- **python:static-inline**: 11 pass, 16 pass*, 0 fail, 0 unsupported -- **python:static-observe**: 27 pass, 0 pass*, 0 fail, 0 unsupported +- **python:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported +- **python:run_async**: 30 pass, 0 pass*, 0 fail, 0 unsupported +- **python:static-steps**: 11 pass, 19 pass*, 0 fail, 0 unsupported +- **python:static-inline**: 11 pass, 19 pass*, 0 fail, 0 unsupported +- **python:static-observe**: 29 pass, 1 pass*, 0 fail, 0 unsupported +- **typescript:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported +- **typescript:static**: 11 pass, 19 pass*, 0 fail, 0 unsupported +- **rust:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported -✅* passed everything the target can observe; it does not expose State or Hooks (the fast static export). ➖ the target cannot express the case. +✅* passed everything the target can observe; it does not expose State or Hooks (the fast mode). ➖ the target cannot express the case. diff --git a/conformance/README.md b/conformance/README.md index f9a852e..6ce43fe 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -4,7 +4,15 @@ The rules are in **[API.md](API.md)**, the CUP API contract. This suite checks them. One set of language-neutral cases, a runner, and **one judge** (`run.py`). -The dynamic run and each static export are held to exactly the same +The judge is itself a codeupipe pipeline, in [judging/](judging/): one step +reads the cases, one runs each target, one compares, and one writes the +matrix. Each step is tested alone in `tests/conformance/test_judge_pipeline.py`. + +The Python runner is built the same way, in +[runners/python/case_runner/](runners/python/case_runner/): one pipeline plays +a case and one times a scenario. Its steps are tested alone in +`tests/conformance/test_case_runner.py`. +The dynamic run and each compiled form are held to exactly the same comparison. ```bash @@ -19,14 +27,17 @@ The committed snapshot is [MATRIX.md](MATRIX.md). ([BENCH.md](BENCH.md)). The runner has a `bench` mode that times the scenarios in `bench/scenarios.json`. -**The speed check** compares each benchmark number with its recorded -baseline (`bench/baseline.json`): +**Speed is measured, not judged.** CI runs `bench.py` and keeps the numbers +(the job summary and an artifact), and never fails because of them: a shared +machine is too noisy for a speed gate. A comparison with a recorded baseline +(`bench/baseline.json`) is kept for a machine whose resources are fixed, such +as a constrained container. No test and no CI job runs it: ```bash -python conformance/bench.py --check # fails outside ±25% +python conformance/bench.py --check # outside ±25% of the baseline python conformance/bench.py --check --tolerance 15 -python conformance/bench.py --record # after a change in speed that you accept -python -m pytest -m perf # the same check, as a test +python conformance/bench.py --record # record the baseline +python conformance/bench.py rust:run --record # only the Rust numbers ``` A number is kept as a ratio to a reference workload (plain dict writes @@ -34,14 +45,19 @@ behind function calls) that the runner times beside it, so the check does not depend on how fast the machine is. Slower than the band is a regression. Faster than the band also fails, with the instruction to record again: otherwise the next regression would hide inside the gain. There is one -baseline for each Python version. Run it on a quiet machine: under heavy +baseline for each Python version, and one for the Rust run, which does not +depend on Python (the Rust runner times a reference of map writes of its own). +`--record` with targets named replaces only those targets. Run it on a quiet machine: under heavy load a number can leave the band. | Target | How it runs (all offline) | |---|---| | `python:run`, `python:run_async` | `codeupipe.core` Pipeline, sync and async entry points | -| `python:static-steps`, `python:static-inline` | `codeupipe.lower` fast export, loaded from disk (no State or Hooks: ✅\*) | -| `python:static-observe` | `codeupipe.lower` export with `observe=True`: State and Hooks judged too | +| `python:static-steps`, `python:static-inline` | `codeupipe.compile` fast mode, loaded from disk (no State or Hooks: ✅\*) | +| `python:static-observe` | `codeupipe.compile` with `observe=True`: State and Hooks judged too | +| `typescript:run` | `packages/codeupipe-ts` Pipeline. Node runs the `.ts` runner as it is (Node 22.18 or later). With no arguments, `run.py` leaves this target out when the machine has no such Node | +| `typescript:static` | `packages/codeupipe-compile-ts`: the pipeline as plain code, written to a file and imported (no State or Hooks: ✅\*). Needs `npm ci` in that package and a built core package; `run.py` leaves it out when they are not there | +| `rust:run` | `packages/codeupipe-rs` Pipeline. `cargo run --release --offline` builds the runner, which has no dependency to download. With no arguments, `run.py` leaves this target out when the machine has no `cargo` | ## Statuses @@ -50,7 +66,7 @@ load a number can leave the band. | ✅ | every expectation matched | | ✅* | everything the target *can observe* matched. Static code has no State or Hooks, so those checks cannot run | | ❌ | something observable differed | -| ➖ | the target cannot express the case (e.g. export refuses a nested pipeline). Never counted as a pass | +| ➖ | the target cannot express the case (e.g. the compiler refuses a nested pipeline). Never counted as a pass | ## Case format (`cases/*.json`) @@ -69,6 +85,16 @@ load a number can leave the band. "taps": [...], "hooks": ["before:", "before:s1", ...]}} ``` +Three more keys are for the cases about State and runs that overlap: + +- `"given_state": true` in a case: the runner gives the run a new State + (`run(p, state)`), and reports that State as `state`. +- `expect.inner_states`: the State of each run that a `run_again` step + started, in order. +- `expect.pipeline_state`: `state` on the Pipeline after the run ends. A + compiled form is a function and has no Pipeline object, so it cannot + show this. + Every case also checks, implicitly, that **the caller's input was not mutated**. Numbers compare as JSON numbers (`1 == 1.0`), but `true` is never `1`. @@ -85,11 +111,12 @@ Numbers compare as JSON numbers (`1 == 1.0`), but `true` is never `1`. | filter | `merge` value | `merge(Payload(value))` | | filter | `has` key, to | insert `to = has(key)` | | filter | `mutate` key, value | `insert(key, value)` as a statement of its own, then return the payload | +| filter | `run_again` input | when `get("inner")` is not true: run the pipeline that this step is in one more time, with `input` and a new State, and keep that State for the report. Then return the payload with no change. This is a second run that starts and ends while the first one is in progress. How a pipeline is run again depends on the target (a method of the Pipeline, or for a compiled form a second call of its function), so the runner gives the step that as a function | | tap | `record` key | append `get(key)` to the runner's tap log | | tap | `fail` message | raise | | valve `when` | `equals` key, value | `get(key) == value` | | valve `when` | `truthy` key | `get(key)` is truthy (missing ⇒ false) | -| hooks | `"hooks": true` | record `before:`, `after:`, `error:`; pipeline-level events use an empty step name. On a filter failure: `error:` then `error:`. The runner maps each reported step identity (API.md) back to the case's step name | +| hooks | `"hooks": true` | record `before:`, `after:`, `error:`; pipeline-level events use an empty step name. On a filter failure: `error:` then `error:`. The runner maps each reported step label (API.md) back to the case's step name | ## Runner protocol @@ -115,4 +142,9 @@ never halts the pipeline). Where it is silent, they encode Python 2. **Hooks around a skipped Valve.** Python fires `before:v` / `after:v` even when the Valve skips its filter. 3. **`equals` across types.** Python's `==` makes `1 == true`; JSON does not. - No case depends on it yet. + TypeScript uses `===`, which does not compare lists or maps by content. + Rust compares `Value` by content, and a number is never equal to a + boolean. No case depends on it yet. +4. **`truthy` across languages.** An empty list or map is false in Python + and true in JavaScript. Each runner uses the rule of its language; Rust + has no rule of its own and follows Python. No case depends on it yet. diff --git a/conformance/bench.py b/conformance/bench.py index 52f0318..6449b15 100644 --- a/conformance/bench.py +++ b/conformance/bench.py @@ -11,7 +11,11 @@ output it produced; this script refuses a result whose output differs from the others, so a fast wrong answer cannot win. -The speed check (--check). A recorded time is true for one machine, so each +The comparison (--check) is for a machine whose resources are fixed, such as +a constrained container. CI does not run it: a shared machine is too noisy +to judge speed, so CI only measures and keeps the numbers. + +A recorded time is true for one machine, so each number is kept as a ratio to a reference workload: plain Python dict writes and function calls, the work a pipeline does. The runner times the reference and the scenario back to back, so a slower or a busier machine slows both @@ -21,7 +25,8 @@ the baseline is out of date and must be recorded again, or the next regression would hide inside the gain. A baseline is kept for each Python version (bench/baseline.json), because interpreters differ in what they make -fast. +fast. The Rust run does not depend on Python, so it has one baseline, under +"rust". --record replaces only the targets that it measured. """ import json @@ -41,6 +46,7 @@ "python:static-steps": ("python:static-steps", "bench:static-steps"), "python:static-inline": ("python:static-inline", "bench:static-inline"), "python:static-observe": ("python:static-observe", "bench:static-observe"), + "rust:run": ("rust:run", "bench:run"), } @@ -57,10 +63,39 @@ def python_key(): return "%d.%d" % sys.version_info[:2] +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def load_baseline(): return json.loads(BASELINE.read_text()) if BASELINE.exists() else {} +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def baseline_key(target, python): + """Where the baseline of ``target`` is kept. A Python target is kept per + Python version; the Rust run does not depend on Python and has its own.""" + return "rust" if target.startswith("rust:") else python + + +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def baseline_for(baseline, targets, python): + """The recorded ratios of ``targets``, each from its own key.""" + relative = {} + for target in targets: + row = baseline.get(baseline_key(target, python), {}).get("relative", {}).get(target) + if row is not None: + relative[target] = row + return {"relative": relative} + + +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline +def record_into(baseline, current, python): + """Put the measured targets into ``baseline``. The others are kept.""" + for kind in ("relative", "us_per_run"): + for target, row in current[kind].items(): + recorded = baseline.setdefault(baseline_key(target, python), {"relative": {}, "us_per_run": {}}) + recorded.setdefault(kind, {})[target] = row + + +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def compare(baseline, current, tolerance=DEFAULT_TOLERANCE): """Every number that is not within ``tolerance`` percent of its baseline. @@ -84,6 +119,7 @@ def compare(baseline, current, tolerance=DEFAULT_TOLERANCE): return problems +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def _report(problems, tolerance): if not problems: return "Speed check: every number is within %g%% of its baseline." % tolerance @@ -101,6 +137,7 @@ def _report(problems, tolerance): return "\n".join(lines) +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def main(argv): argv = list(argv) record, check = "--record" in argv, "--check" in argv @@ -112,7 +149,8 @@ def main(argv): argv = [a for a in argv if a not in ("--record", "--check")] scenarios = json.loads((HERE / "bench" / "scenarios.json").read_text()) - names = argv or list(BENCH_TARGETS) + runnable = conformance.default_targets() + names = argv or [name for name in BENCH_TARGETS if BENCH_TARGETS[name][0] in runnable] table, relative, reference, problems, refused = {}, {}, {}, [], [] for name in names: results, error = conformance.run_target(name, spec_for(name), scenarios) @@ -151,22 +189,23 @@ def main(argv): if record: baseline = load_baseline() - baseline[python_key()] = current + record_into(baseline, current, python_key()) BASELINE.write_text(json.dumps(baseline, indent=2, sort_keys=True) + "\n") - print("Recorded the baseline for Python %s in %s." % (python_key(), BASELINE.relative_to(HERE.parent))) + print("Recorded the baseline of %s in %s." % (", ".join(names), BASELINE.relative_to(HERE.parent))) return 0 if check: - baseline = load_baseline().get(python_key()) - if baseline is None: + baseline = load_baseline() + if python_key() not in baseline and any(name.startswith("python:") for name in names): print("Speed check: no baseline for Python %s. Record one: python conformance/bench.py --record" % python_key()) return 1 - found = compare(baseline, current, tolerance) + found = compare(baseline_for(baseline, names, python_key()), current, tolerance) print(_report(found, tolerance)) return 1 if found else 0 return 0 +# cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def markdown(scenarios, table, problems, refused=()): targets = list(table) lines = ["µs per run (best of 5 × 20,000, after 20,000 warmup runs); lower is better.", "", diff --git a/conformance/bench/baseline.json b/conformance/bench/baseline.json index c903c9f..a4fb61c 100644 --- a/conformance/bench/baseline.json +++ b/conformance/bench/baseline.json @@ -114,5 +114,25 @@ "valve-gated": 1.7382603999999968 } } + }, + "rust": { + "relative": { + "rust:run": { + "heavy-allocation": 5.329424607074787, + "hook": 0.36049069077903273, + "micro-pass": 0.3347908632428459, + "taps": 0.37290526661608375, + "valve-gated": 0.40713696452472475 + } + }, + "us_per_run": { + "rust:run": { + "heavy-allocation": 4.31750625, + "hook": 0.2910354, + "micro-pass": 0.2790875, + "taps": 0.29723954999999996, + "valve-gated": 0.33209375 + } + } } } diff --git a/conformance/cases/state.json b/conformance/cases/state.json new file mode 100644 index 0000000..6bf7701 --- /dev/null +++ b/conformance/cases/state.json @@ -0,0 +1,202 @@ +[ + { + "id": "state/the-caller-gives-the-state-of-a-run", + "given_state": true, + "steps": [ + { + "kind": "filter", + "name": "first", + "op": "set", + "key": "a", + "value": 1 + }, + { + "kind": "valve", + "name": "gate", + "when": { + "op": "truthy", + "key": "inner" + }, + "filter": { + "op": "set", + "key": "gated", + "value": true + } + }, + { + "kind": "tap", + "name": "audit", + "op": "fail", + "message": "sink down" + }, + { + "kind": "filter", + "name": "last", + "op": "add", + "key": "n", + "by": 1 + } + ], + "input": {}, + "expect": { + "output": { + "a": 1, + "n": 1 + }, + "state": { + "executed": [ + "first", + "audit", + "last" + ], + "skipped": [ + "gate" + ], + "errors": [ + "audit" + ] + }, + "inner_states": [] + } + }, + { + "id": "state/a-run-that-starts-inside-another-run-has-its-own-state", + "given_state": true, + "steps": [ + { + "kind": "filter", + "name": "first", + "op": "set", + "key": "a", + "value": 1 + }, + { + "kind": "filter", + "name": "again", + "op": "run_again", + "input": { + "inner": true + } + }, + { + "kind": "valve", + "name": "gate", + "when": { + "op": "truthy", + "key": "inner" + }, + "filter": { + "op": "set", + "key": "gated", + "value": true + } + }, + { + "kind": "filter", + "name": "last", + "op": "add", + "key": "n", + "by": 1 + } + ], + "input": {}, + "expect": { + "output": { + "a": 1, + "n": 1 + }, + "state": { + "executed": [ + "first", + "again", + "last" + ], + "skipped": [ + "gate" + ], + "errors": [] + }, + "inner_states": [ + { + "executed": [ + "first", + "again", + "gate", + "last" + ], + "skipped": [], + "errors": [] + } + ] + } + }, + { + "id": "state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last", + "steps": [ + { + "kind": "filter", + "name": "first", + "op": "set", + "key": "a", + "value": 1 + }, + { + "kind": "filter", + "name": "again", + "op": "run_again", + "input": { + "inner": true + } + }, + { + "kind": "valve", + "name": "gate", + "when": { + "op": "truthy", + "key": "inner" + }, + "filter": { + "op": "set", + "key": "gated", + "value": true + } + }, + { + "kind": "filter", + "name": "last", + "op": "add", + "key": "n", + "by": 1 + } + ], + "input": {}, + "expect": { + "output": { + "a": 1, + "n": 1 + }, + "inner_states": [ + { + "executed": [ + "first", + "again", + "gate", + "last" + ], + "skipped": [], + "errors": [] + } + ], + "pipeline_state": { + "executed": [ + "first", + "again", + "gate", + "last" + ], + "skipped": [], + "errors": [] + } + } + } +] diff --git a/conformance/judging/__init__.py b/conformance/judging/__init__.py new file mode 100644 index 0000000..7e322bb --- /dev/null +++ b/conformance/judging/__init__.py @@ -0,0 +1,5 @@ +"""The judge of the conformance suite, as a codeupipe pipeline (see ``judge_pipeline.py``). + +A runner executes the cases and reports what happened. Only the judge +decides pass or fail, so every target is held to the same comparison. +""" diff --git a/conformance/judging/cargo_probe.py b/conformance/judging/cargo_probe.py new file mode 100644 index 0000000..ecb029c --- /dev/null +++ b/conformance/judging/cargo_probe.py @@ -0,0 +1,10 @@ +"""CargoProbe: what this machine has for the Rust targets.""" + +import shutil + + +class CargoProbe: + """Says if there is a ``cargo`` to build and run the Rust runner.""" + + def present(self) -> bool: + return shutil.which("cargo") is not None diff --git a/conformance/judging/default_targets_filter.py b/conformance/judging/default_targets_filter.py new file mode 100644 index 0000000..44588fb --- /dev/null +++ b/conformance/judging/default_targets_filter.py @@ -0,0 +1,34 @@ +"""DefaultTargetsFilter: each target that this machine can run.""" + +from codeupipe import Payload + + +class DefaultTargetsFilter: + """Filter (sync). + + Input keys: ``targets`` ({name: how to start its runner}), ``needs`` + ({name: paths that must exist for the target}). + Output keys: ``names``. A Python target always runs. A TypeScript target + needs a Node that runs ``.ts`` files, and the paths of ``needs``. A Rust + target needs ``cargo``. + + ``node_runs_typescript`` and ``has_cargo`` are functions with no + arguments that say if the machine has such a Node, and a ``cargo``. + """ + + def __init__(self, node_runs_typescript, has_cargo=lambda: False): + self.node_runs_typescript = node_runs_typescript + self.has_cargo = has_cargo + + def call(self, payload: Payload) -> Payload: + return payload.insert("names", self.choose(payload.get("targets"), payload.get("needs", {}))) + + def choose(self, targets, needs) -> list: + return [name for name in targets if self.can_run(name, needs)] + + def can_run(self, name, needs) -> bool: + if name.startswith("rust:"): + return self.has_cargo() + if not name.startswith("typescript:"): + return True + return self.node_runs_typescript() and all(path.exists() for path in needs.get(name, [])) diff --git a/conformance/judging/format_matrix_filter.py b/conformance/judging/format_matrix_filter.py new file mode 100644 index 0000000..3d54bfa --- /dev/null +++ b/conformance/judging/format_matrix_filter.py @@ -0,0 +1,32 @@ +"""FormatMatrixFilter: the matrix and the counts as Markdown.""" + +from codeupipe import Payload + +_MARK = {"pass": "✅", "partial": "✅*", "FAIL": "❌", "unsupported": "➖", "—": "—"} + + +class FormatMatrixFilter: + """Filter (sync). + + Input keys: ``cases``, ``names``, ``matrix``, ``report``. + Output keys: ``markdown``. + """ + + def call(self, payload: Payload) -> Payload: + cases, names = payload.get("cases"), payload.get("names") + matrix, report = payload.get("matrix"), payload.get("report") + lines = ["| case | " + " | ".join(names) + " |", "|---|" + "---|" * len(names)] + for case in cases: + lines.append(f"| `{case['id']}` | " + " | ".join(_MARK[matrix[case['id']][n]] for n in names) + " |") + lines.append("") + for name in names: + target = report["targets"][name] + if "error" in target: + lines.append(f"- **{name}**: not run: {target['error']}") + else: + lines.append(f"- **{name}**: {target['pass']} pass, {target['partial']} pass*, {target['FAIL']} fail, " + f"{target['unsupported']} unsupported") + lines.append("") + lines.append("✅* passed everything the target can observe; it does not expose State or Hooks " + "(the fast mode). ➖ the target cannot express the case.") + return payload.insert("markdown", "\n".join(lines) + "\n") diff --git a/conformance/judging/judge_pipeline.py b/conformance/judging/judge_pipeline.py new file mode 100644 index 0000000..87afcb0 --- /dev/null +++ b/conformance/judging/judge_pipeline.py @@ -0,0 +1,37 @@ +"""JudgePipeline: run the conformance cases on each target and judge them one way.""" + +from codeupipe import Pipeline + +from .default_targets_filter import DefaultTargetsFilter +from .format_matrix_filter import FormatMatrixFilter +from .judge_results_filter import JudgeResultsFilter +from .load_cases_filter import LoadCasesFilter +from .no_targets_named_predicate import NoTargetsNamedPredicate +from .run_targets_filter import RunTargetsFilter +from .summarize_filter import SummarizeFilter +from .write_results_tap import WriteResultsTap + + +class JudgePipeline(Pipeline): + """The judge. + + Input keys: ``cases_dir``, ``results_dir``, ``targets`` ({name: how to + start its runner}), ``needs`` ({name: paths that the target needs}), and + ``names`` (the targets that the caller named; empty for each target that + this machine can run). + Output keys: ``report``, ``matrix``, ``markdown``, ``exit_code``. + + ``launcher`` starts the runner of a target. ``node_runs_typescript`` says + if the machine can run the TypeScript targets, and ``has_cargo`` if it + can run the Rust targets. + """ + + def __init__(self, launcher, node_runs_typescript, has_cargo=lambda: False): + super().__init__() + self.add_filter(LoadCasesFilter()) + self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript, has_cargo)) + self.add_filter(RunTargetsFilter(launcher)) + self.add_filter(JudgeResultsFilter()) + self.add_filter(SummarizeFilter()) + self.add_filter(FormatMatrixFilter()) + self.add_tap(WriteResultsTap()) diff --git a/conformance/judging/judge_results_filter.py b/conformance/judging/judge_results_filter.py new file mode 100644 index 0000000..1f9a43c --- /dev/null +++ b/conformance/judging/judge_results_filter.py @@ -0,0 +1,93 @@ +"""JudgeResultsFilter: compare what each target reported with what each case expects. + +This is the one place that decides pass or fail. A verdict is a status and +its problems: + + pass every expectation matched + partial every expectation that the target can show matched, but it + does not show some of them at all (the fast compiled forms + show no State and no Hooks) + FAIL something that the target shows is different + unsupported the target cannot express the case. Never counted as a pass +""" + +from codeupipe import Payload + +# What a target may not show at all. A case that expects one of these, on a target that does not report it, is "partial". +_NOT_ALWAYS_SHOWN = ("state", "hooks", "inner_states", "pipeline_state") + + +class JudgeResultsFilter: + """Filter (sync). + + Input keys: ``cases``, ``results`` ({name: {case id: result}}). + Output keys: ``verdicts`` ({name: {case id: (status, problems)}}). + """ + + def call(self, payload: Payload) -> Payload: + cases = payload.get("cases") + verdicts = {name: {case["id"]: self.judge(case, found.get(case["id"])) for case in cases} + for name, found in payload.get("results").items()} + return payload.insert("verdicts", verdicts) + + def judge(self, case, result): + """The verdict for one case on one target: ``(status, problems)``.""" + if result is None: + return "FAIL", ["no result from runner"] + if "unsupported" in result: + return "unsupported", [result["unsupported"]] + expect = case["expect"] + problems = self.output_problems(expect, result) + + if self.norm(result.get("input_after")) != self.norm(case["input"]): + problems.append(f"input was mutated: {result.get('input_after')!r}") + + if "taps" in expect and self.norm(result.get("taps")) != self.norm(expect["taps"]): + problems.append(f"taps {result.get('taps')!r} != {expect['taps']!r}") + + if "state" in expect and "state" in result: + for key, want in expect["state"].items(): + got = result["state"].get(key) + if got != want: + problems.append(f"state.{key} {got!r} != {want!r}") + + for key in ("inner_states", "pipeline_state"): + if key in expect and key in result and result[key] != expect[key]: + problems.append(f"{key} {result[key]!r} != {expect[key]!r}") + + if "hooks" in expect and "hooks" in result and result["hooks"] != expect["hooks"]: + problems.append(f"hooks {result['hooks']!r} != {expect['hooks']!r}") + + if problems: + return "FAIL", problems + unchecked = [key for key in _NOT_ALWAYS_SHOWN if key in expect and key not in result] + if unchecked: + return "partial", [f"target cannot observe {', '.join(unchecked)}"] + return "pass", [] + + def output_problems(self, expect, result) -> list: + """What is wrong with the output or the error of a run.""" + problems = [] + if "output" in expect: + if "error" in result: + problems.append(f"expected output, got error {result['error']!r}") + elif self.norm(result.get("output")) != self.norm(expect["output"]): + problems.append(f"output {result.get('output')!r} != {expect['output']!r}") + if "error" in expect: + if "error" not in result: + problems.append(f"expected an error, got output {result.get('output')!r}") + elif expect["error"].get("message_contains", "") not in result["error"]: + problems.append(f"error {result['error']!r} lacks {expect['error']['message_contains']!r}") + return problems + + def norm(self, value): + """JSON equality: 1 == 1.0, but true is not 1.""" + if isinstance(value, bool) or value is None or isinstance(value, str): + return value + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, list): + return [self.norm(v) for v in value] + if isinstance(value, dict): + return {k: self.norm(v) for k, v in value.items()} + return value diff --git a/conformance/judging/load_cases_filter.py b/conformance/judging/load_cases_filter.py new file mode 100644 index 0000000..b3649f7 --- /dev/null +++ b/conformance/judging/load_cases_filter.py @@ -0,0 +1,28 @@ +"""LoadCasesFilter: read the conformance cases from disk.""" + +import json +import pathlib + +from codeupipe import Payload + + +class LoadCasesFilter: + """Filter (sync). + + Input keys: ``cases_dir`` (the directory of the ``*.json`` case files). + Output keys: ``cases`` (each case, in the order of the file names). Two + cases with the same id stop the run. + """ + + def call(self, payload: Payload) -> Payload: + return payload.insert("cases", self.load(payload.get("cases_dir"))) + + def load(self, cases_dir) -> list: + cases = [] + for path in sorted(pathlib.Path(cases_dir).glob("*.json")): + cases.extend(json.loads(path.read_text())) + ids = [case["id"] for case in cases] + duplicates = {i for i in ids if ids.count(i) > 1} + if duplicates: + raise SystemExit(f"duplicate case ids: {sorted(duplicates)}") + return cases diff --git a/conformance/judging/no_targets_named_predicate.py b/conformance/judging/no_targets_named_predicate.py new file mode 100644 index 0000000..9adbeda --- /dev/null +++ b/conformance/judging/no_targets_named_predicate.py @@ -0,0 +1,10 @@ +"""NoTargetsNamedPredicate: did the caller name no target?""" + +from codeupipe import Payload + + +class NoTargetsNamedPredicate: + """True when ``names`` is empty: the judge then takes each target that this machine can run.""" + + def test(self, payload: Payload) -> bool: + return not payload.get("names") diff --git a/conformance/judging/node_probe.py b/conformance/judging/node_probe.py new file mode 100644 index 0000000..80bd764 --- /dev/null +++ b/conformance/judging/node_probe.py @@ -0,0 +1,20 @@ +"""NodeProbe: what this machine has for the TypeScript targets.""" + +import shutil +import subprocess + + +# cup: allow-plain the edge to the machine: it asks which Node is installed +class NodeProbe: + """Says if there is a Node that runs a ``.ts`` file as it is (22.18 or later).""" + + def runs_typescript(self) -> bool: + node = shutil.which("node") + if node is None: + return False + try: + version = subprocess.run([node, "--version"], capture_output=True, text=True, timeout=30).stdout + major, minor = (int(part) for part in version.strip().lstrip("v").split(".")[:2]) + except (OSError, ValueError, subprocess.SubprocessError): + return False + return (major, minor) >= (22, 18) diff --git a/conformance/judging/process_launcher.py b/conformance/judging/process_launcher.py new file mode 100644 index 0000000..70b76f8 --- /dev/null +++ b/conformance/judging/process_launcher.py @@ -0,0 +1,29 @@ +"""ProcessLauncher: starts the runner of one target and reads what it reports.""" + +import json +import os +import subprocess + + +# cup: allow-plain the edge to the runners: it starts a process, gives it the cases and reads its report +class ProcessLauncher: + """Runs one target. ``spec`` has ``cmd``, ``env`` and ``cwd``. + + Gives ``(results, None)``, with ``results`` as {case id: result}, or + ``(None, reason)`` when the runner did not end well or its output + cannot be read. + """ + + def run(self, spec, cases): + env = dict(os.environ) + env.update(spec["env"]) + proc = subprocess.run( + spec["cmd"], input=json.dumps(cases), capture_output=True, text=True, + cwd=spec["cwd"], env=env, timeout=600, + ) + if proc.returncode != 0: + return None, f"runner exited {proc.returncode}: {proc.stderr.strip()[-2000:]}" + try: + return {r["id"]: r for r in json.loads(proc.stdout)}, None + except (json.JSONDecodeError, KeyError) as error: + return None, f"unreadable runner output ({error}): {proc.stdout[:500]}" diff --git a/conformance/judging/run_targets_filter.py b/conformance/judging/run_targets_filter.py new file mode 100644 index 0000000..2f7c56e --- /dev/null +++ b/conformance/judging/run_targets_filter.py @@ -0,0 +1,29 @@ +"""RunTargetsFilter: run the cases on each target.""" + +from codeupipe import Payload + + +class RunTargetsFilter: + """Filter (sync). + + Input keys: ``names`` (the targets to run, in order), ``targets`` + ({name: how to start its runner}), ``cases``. + Output keys: ``results`` ({name: {case id: result}}) for each target that + ran, and ``target_errors`` ({name: reason}) for each one that did not. + + ``launcher`` runs one target: ``launcher.run(spec, cases)``. + """ + + def __init__(self, launcher): + self.launcher = launcher + + def call(self, payload: Payload) -> Payload: + targets, cases = payload.get("targets"), payload.get("cases") + results, errors = {}, {} + for name in payload.get("names"): + found, error = self.launcher.run(targets[name], cases) + if error: + errors[name] = error + else: + results[name] = found + return payload.insert("results", results).insert("target_errors", errors) diff --git a/conformance/judging/summarize_filter.py b/conformance/judging/summarize_filter.py new file mode 100644 index 0000000..2d060cc --- /dev/null +++ b/conformance/judging/summarize_filter.py @@ -0,0 +1,37 @@ +"""SummarizeFilter: the counts for each target, the matrix, and the exit code.""" + +from codeupipe import Payload + + +class SummarizeFilter: + """Filter (sync). + + Input keys: ``cases``, ``names``, ``verdicts``, ``target_errors``. + Output keys: ``report`` (the number of cases, and for each target its + counts, its failures and the cases that it cannot express, or the + reason that it did not run), ``matrix`` ({case id: {name: status}}) and + ``exit_code`` (1 when a case failed or a target did not run). + """ + + def call(self, payload: Payload) -> Payload: + cases, verdicts, errors = payload.get("cases"), payload.get("verdicts"), payload.get("target_errors") + report = {"cases": len(cases), "targets": {}} + matrix = {case["id"]: {} for case in cases} + for name in payload.get("names"): + if name in errors: + report["targets"][name] = {"error": errors[name]} + for case in cases: + matrix[case["id"]][name] = "—" + continue + summary = {"pass": 0, "partial": 0, "FAIL": 0, "unsupported": 0, "failures": {}, "unsupported_cases": {}} + for case in cases: + status, problems = verdicts[name][case["id"]] + summary[status] += 1 + matrix[case["id"]][name] = status + if status == "FAIL": + summary["failures"][case["id"]] = problems + elif status == "unsupported": + summary["unsupported_cases"][case["id"]] = problems[0] + report["targets"][name] = summary + failed = any(target.get("FAIL") or "error" in target for target in report["targets"].values()) + return payload.insert("report", report).insert("matrix", matrix).insert("exit_code", 1 if failed else 0) diff --git a/conformance/judging/write_results_tap.py b/conformance/judging/write_results_tap.py new file mode 100644 index 0000000..5bacf03 --- /dev/null +++ b/conformance/judging/write_results_tap.py @@ -0,0 +1,20 @@ +"""WriteResultsTap: put the report and the matrix on disk.""" + +import json +import pathlib + +from codeupipe import Payload + + +class WriteResultsTap: + """Tap (sync). + + Reads ``results_dir``, ``report`` and ``markdown``, and writes + ``latest.json`` and ``latest.md`` in that directory. + """ + + def observe(self, payload: Payload) -> None: + directory = pathlib.Path(payload.get("results_dir")) + directory.mkdir(exist_ok=True) + (directory / "latest.json").write_text(json.dumps(payload.get("report"), indent=2)) + (directory / "latest.md").write_text(payload.get("markdown")) diff --git a/conformance/run.py b/conformance/run.py index 1a11a15..126b86f 100644 --- a/conformance/run.py +++ b/conformance/run.py @@ -2,6 +2,12 @@ python conformance/run.py # all targets python conformance/run.py python:run python:static-inline + python conformance/run.py typescript:run + python conformance/run.py rust:run + +With no arguments, the TypeScript target runs only when the machine has +Node 22.18 or later, and the Rust target only when it has cargo. A target +that is named and cannot run is a failure. A runner executes cases and reports what happened. Only this file decides pass or fail, so every target is held to exactly the same comparison. @@ -11,20 +17,28 @@ partial every expectation the target can observe matched, but some (State, Hooks) it does not expose at all (fast static code) FAIL something observable differed (details in the JSON report) - unsupported the target cannot express the case (e.g. static export of a + unsupported the target cannot express the case (e.g. compiled form of a nested pipeline); reported, never counted as a pass """ -import json -import os import pathlib -import subprocess import sys HERE = pathlib.Path(__file__).resolve().parent REPO = HERE.parent CASES = HERE / "cases" RESULTS = HERE / "results" +sys.path.insert(0, str(HERE)) + +from codeupipe import Payload # noqa: E402 + +from judging.cargo_probe import CargoProbe # noqa: E402 +from judging.default_targets_filter import DefaultTargetsFilter # noqa: E402 +from judging.judge_pipeline import JudgePipeline # noqa: E402 +from judging.judge_results_filter import JudgeResultsFilter # noqa: E402 +from judging.load_cases_filter import LoadCasesFilter # noqa: E402 +from judging.node_probe import NodeProbe # noqa: E402 +from judging.process_launcher import ProcessLauncher # noqa: E402 PYTHON = sys.executable @@ -37,146 +51,89 @@ def _python(target): } +def _typescript(target): + # Node runs the .ts file as it is. Nothing is built or downloaded. + return { + "cmd": ["node", str(HERE / "runners/typescript/runner.ts"), target], + "env": {}, + "cwd": REPO, + } + + +def _rust(target): + # Built offline from the repository: the runner has no dependency to download. + return { + "cmd": ["cargo", "run", "--quiet", "--release", "--offline", + "--manifest-path", str(HERE / "runners/rust/Cargo.toml"), "--", target], + "env": {}, + "cwd": REPO, + } + + TARGETS = { "python:run": _python("run"), "python:run_async": _python("run_async"), "python:static-steps": _python("static-steps"), "python:static-inline": _python("static-inline"), "python:static-observe": _python("static-observe"), + "typescript:run": _typescript("run"), + "typescript:static": _typescript("static"), + "rust:run": _rust("run"), +} + +# The static target runs the compile package. Its dependencies must be +# installed (npm ci at the repository root), and it uses the built core +# package. +_TYPESCRIPT_NEEDS = { + "typescript:static": [REPO / "node_modules/typescript", + REPO / "packages/codeupipe-ts/dist/index.js"], } +# The judge is the pipeline in judging/. The functions below give its parts to the tests and to bench.py +# under the names that they had. Each one calls a step of the pipeline, so there is one judge. + +def _node_runs_typescript(): + """Whether there is a Node that runs a .ts file as it is (22.18 or later).""" + return NodeProbe().runs_typescript() + + +def _has_cargo(): + """Whether there is a cargo to build and run the Rust runner.""" + return CargoProbe().present() + + +def default_targets(): + """Every target that this machine can run. TypeScript needs Node, the + static TypeScript target needs its package to be installed, and Rust + needs cargo.""" + return DefaultTargetsFilter(lambda: _node_runs_typescript(), lambda: _has_cargo()).choose(TARGETS, _TYPESCRIPT_NEEDS) + + def load_cases(): - cases = [] - for path in sorted(CASES.glob("*.json")): - cases.extend(json.loads(path.read_text())) - ids = [c["id"] for c in cases] - duplicates = {i for i in ids if ids.count(i) > 1} - if duplicates: - raise SystemExit(f"duplicate case ids: {sorted(duplicates)}") - return cases + return LoadCasesFilter().load(CASES) def run_target(name, spec, cases): - env = dict(os.environ) - env.update(spec["env"]) - proc = subprocess.run( - spec["cmd"], input=json.dumps(cases), capture_output=True, text=True, - cwd=spec["cwd"], env=env, timeout=600, - ) - if proc.returncode != 0: - return None, f"runner exited {proc.returncode}: {proc.stderr.strip()[-2000:]}" - try: - return {r["id"]: r for r in json.loads(proc.stdout)}, None - except (json.JSONDecodeError, KeyError) as error: - return None, f"unreadable runner output ({error}): {proc.stdout[:500]}" + return ProcessLauncher().run(spec, cases) def _norm(value): """JSON equality: 1 == 1.0, but true is not 1.""" - if isinstance(value, bool) or value is None or isinstance(value, str): - return value - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, list): - return [_norm(v) for v in value] - if isinstance(value, dict): - return {k: _norm(v) for k, v in value.items()} - return value + return JudgeResultsFilter().norm(value) def judge(case, result): """Return (status, problems).""" - if result is None: - return "FAIL", ["no result from runner"] - if "unsupported" in result: - return "unsupported", [result["unsupported"]] - expect = case["expect"] - problems = [] - - if "output" in expect: - if "error" in result: - problems.append(f"expected output, got error {result['error']!r}") - elif _norm(result.get("output")) != _norm(expect["output"]): - problems.append(f"output {result.get('output')!r} != {expect['output']!r}") - if "error" in expect: - if "error" not in result: - problems.append(f"expected an error, got output {result.get('output')!r}") - elif expect["error"].get("message_contains", "") not in result["error"]: - problems.append(f"error {result['error']!r} lacks {expect['error']['message_contains']!r}") - - if _norm(result.get("input_after")) != _norm(case["input"]): - problems.append(f"input was mutated: {result.get('input_after')!r}") - - if "taps" in expect and _norm(result.get("taps")) != _norm(expect["taps"]): - problems.append(f"taps {result.get('taps')!r} != {expect['taps']!r}") - - if "state" in expect and "state" in result: - for key, want in expect["state"].items(): - got = result["state"].get(key) - if got != want: - problems.append(f"state.{key} {got!r} != {want!r}") - - if "hooks" in expect and "hooks" in result and result["hooks"] != expect["hooks"]: - problems.append(f"hooks {result['hooks']!r} != {expect['hooks']!r}") - - if problems: - return "FAIL", problems - unchecked = [k for k in ("state", "hooks") if k in expect and k not in result] - if unchecked: - return "partial", [f"target cannot observe {', '.join(unchecked)}"] - return "pass", [] + return JudgeResultsFilter().judge(case, result) def main(argv): - cases = load_cases() - names = argv or list(TARGETS) - report = {"cases": len(cases), "targets": {}} - matrix = {c["id"]: {} for c in cases} - - for name in names: - results, error = run_target(name, TARGETS[name], cases) - if error: - report["targets"][name] = {"error": error} - for c in cases: - matrix[c["id"]][name] = "—" - continue - summary = {"pass": 0, "partial": 0, "FAIL": 0, "unsupported": 0, "failures": {}, "unsupported_cases": {}} - for c in cases: - status, problems = judge(c, results.get(c["id"])) - summary[status] += 1 - matrix[c["id"]][name] = status - if status == "FAIL": - summary["failures"][c["id"]] = problems - elif status == "unsupported": - summary["unsupported_cases"][c["id"]] = problems[0] - report["targets"][name] = summary - - RESULTS.mkdir(exist_ok=True) - (RESULTS / "latest.json").write_text(json.dumps(report, indent=2)) - (RESULTS / "latest.md").write_text(_markdown(cases, names, matrix, report)) - print(_markdown(cases, names, matrix, report)) - failed = any(t.get("FAIL") for t in report["targets"].values()) - return 1 if failed else 0 - - -def _markdown(cases, names, matrix, report): - mark = {"pass": "✅", "partial": "✅*", "FAIL": "❌", "unsupported": "➖", "—": "—"} - lines = ["| case | " + " | ".join(names) + " |", "|---|" + "---|" * len(names)] - for c in cases: - lines.append(f"| `{c['id']}` | " + " | ".join(mark[matrix[c['id']][n]] for n in names) + " |") - lines.append("") - for name in names: - t = report["targets"][name] - if "error" in t: - lines.append(f"- **{name}**: not run: {t['error']}") - else: - lines.append(f"- **{name}**: {t['pass']} pass, {t['partial']} pass*, {t['FAIL']} fail, " - f"{t['unsupported']} unsupported") - lines.append("") - lines.append("✅* passed everything the target can observe; it does not expose State or Hooks " - "(the fast static export). ➖ the target cannot express the case.") - return "\n".join(lines) + "\n" + pipeline = JudgePipeline(ProcessLauncher(), lambda: _node_runs_typescript(), lambda: _has_cargo()) + out = pipeline.run(Payload({"cases_dir": CASES, "results_dir": RESULTS, "targets": TARGETS, + "needs": _TYPESCRIPT_NEEDS, "names": list(argv)})) + print(out.get("markdown")) + return out.get("exit_code") if __name__ == "__main__": diff --git a/conformance/runners/python/case_runner/__init__.py b/conformance/runners/python/case_runner/__init__.py new file mode 100644 index 0000000..a453e76 --- /dev/null +++ b/conformance/runners/python/case_runner/__init__.py @@ -0,0 +1,6 @@ +"""The Python conformance runner, as codeupipe pipelines. + +``run_case_pipeline.py`` plays one case on one target and reports what +happened. ``bench_scenario_pipeline.py`` times one scenario. A runner never +judges: ``conformance/judging`` does. +""" diff --git a/conformance/runners/python/case_runner/attach_noop_hook_filter.py b/conformance/runners/python/case_runner/attach_noop_hook_filter.py new file mode 100644 index 0000000..8170d41 --- /dev/null +++ b/conformance/runners/python/case_runner/attach_noop_hook_filter.py @@ -0,0 +1,19 @@ +"""AttachNoopHookFilter: a Hook that does nothing, for a scenario that measures the cost of Hooks.""" + +import cup_ops as ops +from codeupipe import Payload + + +class AttachNoopHookFilter: + """Filter (sync). + + Input keys: ``case`` (a scenario), ``pipeline``. + Output keys: ``hooks``: a list with one NoopHook for a scenario with + ``"hooks": "noop"``, or None. The pipeline gets its own NoopHook. + """ + + def call(self, payload: Payload) -> Payload: + if payload.get("case").get("hooks") != "noop": + return payload.insert("hooks", None) + payload.get("pipeline").use_hook(ops.NoopHook()) + return payload.insert("hooks", [ops.NoopHook()]) diff --git a/conformance/runners/python/case_runner/attach_record_hook_filter.py b/conformance/runners/python/case_runner/attach_record_hook_filter.py new file mode 100644 index 0000000..314be3b --- /dev/null +++ b/conformance/runners/python/case_runner/attach_record_hook_filter.py @@ -0,0 +1,22 @@ +"""AttachRecordHookFilter: the Hook that records the lifecycle, for a case that asks for it.""" + +import cup_ops as ops +from codeupipe import Payload + + +class AttachRecordHookFilter: + """Filter (sync). + + Input keys: ``case``, ``pipeline``, ``compiled``, ``observe``. + Output keys: ``hooks``: the Hooks of the run, or None. A case with + ``"hooks"`` gets a RecordHook, on the pipeline too. Compiled code calls + Hooks only when it was compiled with ``observe``. + """ + + def call(self, payload: Payload) -> Payload: + wanted = payload.get("case").get("hooks") and (not payload.get("compiled") or payload.get("observe")) + if not wanted: + return payload.insert("hooks", None) + hook = ops.RecordHook() + payload.get("pipeline").use_hook(hook) + return payload.insert("hooks", [hook]) diff --git a/conformance/runners/python/case_runner/bench_each_scenario_filter.py b/conformance/runners/python/case_runner/bench_each_scenario_filter.py new file mode 100644 index 0000000..f176275 --- /dev/null +++ b/conformance/runners/python/case_runner/bench_each_scenario_filter.py @@ -0,0 +1,38 @@ +"""BenchEachScenarioFilter: time each scenario, one after the other.""" + +import pathlib +import tempfile + +from codeupipe import Payload + +from .unsupported import ExportRefused + + +class BenchEachScenarioFilter: + """Filter (sync). + + Input keys: ``cases`` (the scenarios), ``target``. + Output keys: ``results``: for each scenario ``id``, ``us_per_run``, + ``relative`` and ``output``, or ``{"id", "unsupported": reason}`` when + the compiler refuses its pipeline. + + ``each`` is the pipeline that times one scenario. + """ + + def __init__(self, each): + self.each = each + + def call(self, payload: Payload) -> Payload: + target = payload.get("target") + results = [self.one(case, target, index) for index, case in enumerate(payload.get("cases"))] + return payload.insert("results", results) + + def one(self, case, target, index) -> dict: + workdir = pathlib.Path(tempfile.mkdtemp(prefix="cup_bench_")) + try: + out = self.each.run(Payload({"case": case, "target": target, "workdir": workdir, + "module_name": f"bench_predicates_{index}"})) + except ExportRefused as reason: + return {"id": case["id"], "unsupported": str(reason)} + return {"id": case["id"], "us_per_run": out.get("us_per_run"), "relative": out.get("relative"), + "output": out.get("output")} diff --git a/conformance/runners/python/case_runner/bench_scenario_pipeline.py b/conformance/runners/python/case_runner/bench_scenario_pipeline.py new file mode 100644 index 0000000..fd8a41a --- /dev/null +++ b/conformance/runners/python/case_runner/bench_scenario_pipeline.py @@ -0,0 +1,30 @@ +"""BenchScenarioPipeline: time one scenario on one target.""" + +from codeupipe import Pipeline + +from .attach_noop_hook_filter import AttachNoopHookFilter +from .build_pipeline_filter import BuildPipelineFilter +from .export_filter import ExportFilter +from .is_compiled_predicate import IsCompiledPredicate +from .make_run_function_filter import MakeRunFunctionFilter +from .measure_filter import MeasureFilter +from .parse_target_filter import ParseTargetFilter +from .write_predicates_filter import WritePredicatesFilter + +TARGETS = ("run", "static-steps", "static-inline", "static-observe") + + +class BenchScenarioPipeline(Pipeline): + """Input keys: ``case`` (a scenario), ``target``, ``workdir``, ``module_name``. + Output keys: ``us_per_run``, ``relative``, ``output``. + """ + + def __init__(self): + super().__init__() + self.add_filter(ParseTargetFilter(TARGETS, "bench target")) + self.add_filter(WritePredicatesFilter()) + self.add_filter(BuildPipelineFilter()) + self.add_filter(AttachNoopHookFilter()) + self.add_valve(IsCompiledPredicate(), ExportFilter()) + self.add_filter(MakeRunFunctionFilter()) + self.add_filter(MeasureFilter()) diff --git a/conformance/runners/python/case_runner/build_pipeline_filter.py b/conformance/runners/python/case_runner/build_pipeline_filter.py new file mode 100644 index 0000000..d62bb4a --- /dev/null +++ b/conformance/runners/python/case_runner/build_pipeline_filter.py @@ -0,0 +1,79 @@ +"""BuildPipelineFilter: the pipeline of a case, made from its steps.""" + +import cup_ops as ops +from codeupipe import Payload, Pipeline + +from .unsupported import Unsupported + +# The reference vocabulary (conformance/README.md): for each op, how its step is made from the case. +_FILTERS = { + "set": lambda spec: ops.SetFilter(spec["key"], spec["value"]), + "copy": lambda spec: ops.CopyFilter(spec["from"], spec["to"]), + "add": lambda spec: ops.AddFilter(spec["key"], spec["by"]), + "append": lambda spec: ops.AppendFilter(spec["key"], spec["value"]), + "remove": lambda spec: ops.RemoveFilter(spec["key"]), + "merge": lambda spec: ops.MergeFilter(spec["value"]), + "has": lambda spec: ops.HasFilter(spec["key"], spec["to"]), + "mutate": lambda spec: ops.MutateFilter(spec["key"], spec["value"]), + "fail": lambda spec: ops.FailFilter(spec["message"]), + "run_again": lambda spec: ops.RunAgainFilter(spec["input"]), +} +_ASYNC_FILTERS = {**_FILTERS, "run_again": lambda spec: ops.AsyncRunAgainFilter(spec["input"])} +_TAPS = { + "record": lambda spec: ops.RecordTap(spec["key"]), + "fail": lambda spec: ops.FailTap(spec["message"]), + "noop": lambda spec: ops.NoopTap(), +} + + +class BuildPipelineFilter: + """Filter (sync). + + Input keys: ``case``, ``predicates`` (from WritePredicatesFilter), + ``run_async``. + Output keys: ``pipeline``, and ``label``: for each step label that the + pipeline reports, the name that the case gives that step. Steps are + reported by label (conformance/API.md) and the cases name them, and this + Filter built each step, so it knows which is which. A label that is not + in the map stays as it is, and fails the comparison. + """ + + def call(self, payload: Payload) -> Payload: + names = {} + pipeline = self.build(payload.get("case")["steps"], Pipeline, payload.get("predicates"), + names, payload.get("run_async")) + label = {step_label: names[id(step)] for step_label, step, _type in pipeline._steps} + return payload.insert("pipeline", pipeline).insert("label", label) + + def build(self, steps, cls, predicates, names, run_async=False): + pipeline = cls() + for step in steps: + kind, name = step["kind"], step["name"] + if kind == "filter": + made = self.filter(step, run_async) + pipeline.add_filter(made) + elif kind == "tap": + made = self.tap(step) + pipeline.add_tap(made) + elif kind == "valve": + pipeline.add_valve(getattr(predicates, f"when_{name}"), self.filter(step["filter"])) + made = pipeline._steps[-1][1] # the Valve that the pipeline made: the step that the label names + elif kind == "pipeline": + made = self.build(step["steps"], ops.NestedPipeline, predicates, names) + pipeline.add_pipeline(made) + else: + raise Unsupported(f"unknown step kind {kind!r}") + names[id(made)] = name + return pipeline + + def filter(self, spec, run_async=False): + make = (_ASYNC_FILTERS if run_async else _FILTERS).get(spec["op"]) + if make is None: + raise Unsupported(f"unknown filter op {spec['op']!r}") + return make(spec) + + def tap(self, spec): + make = _TAPS.get(spec["op"]) + if make is None: + raise Unsupported(f"unknown tap op {spec['op']!r}") + return make(spec) diff --git a/conformance/runners/python/case_runner/export_filter.py b/conformance/runners/python/case_runner/export_filter.py new file mode 100644 index 0000000..54b04ca --- /dev/null +++ b/conformance/runners/python/case_runner/export_filter.py @@ -0,0 +1,48 @@ +"""ExportFilter: compile the pipeline of the case and import the package from disk, as a customer would.""" + +import importlib +import sys + +from codeupipe import Payload + +from .unsupported import ExportRefused + + +class ExportFilter: + """Filter (sync). + + Input keys: ``pipeline``, ``lowering``, ``observe``, ``workdir``. + Output keys: ``package``: the compiled code, with its ``run`` function. + + The compiler refuses a pipeline that it cannot keep equal. For the fast + forms that is an ExportRefused: the target cannot express the case. With + ``observe`` the error is given as it is. + """ + + def call(self, payload: Payload) -> Payload: + try: + package = self.load(payload.get("pipeline"), payload.get("lowering"), payload.get("observe"), + payload.get("workdir")) + except Exception as error: # noqa: BLE001 — the compiler refused the shape (for example Hooks) + if payload.get("observe"): + raise + raise ExportRefused(f"{type(error).__name__}: {error}") from error + return payload.insert("package", package) + + def load(self, pipeline, lowering, observe, workdir): + from codeupipe.compile import build_export_pipeline_v2 + + out = build_export_pipeline_v2(write=False).run( + Payload({"pipeline": pipeline, "pattern": "flat", "lowering": lowering, "observe": observe})) + for file in out.get("files") or []: + target = workdir / file["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(file["content"]) + package_parent = str(workdir / "src") + for name in [m for m in sys.modules if m == "steps" or m.startswith("steps.")]: + del sys.modules[name] + sys.path.insert(0, package_parent) + try: + return importlib.import_module("steps") + finally: + sys.path.remove(package_parent) diff --git a/conformance/runners/python/case_runner/finish_result_filter.py b/conformance/runners/python/case_runner/finish_result_filter.py new file mode 100644 index 0000000..6f0c991 --- /dev/null +++ b/conformance/runners/python/case_runner/finish_result_filter.py @@ -0,0 +1,16 @@ +"""FinishResultFilter: the id of the case and what its Taps recorded.""" + +import conformance_sink as sink +from codeupipe import Payload + + +class FinishResultFilter: + """Filter (sync). + + Input keys: ``case``, ``result``. + Output keys: ``result``, with ``id`` and ``taps``. + """ + + def call(self, payload: Payload) -> Payload: + result = {**payload.get("result"), "id": payload.get("case")["id"], "taps": list(sink.TAPS)} + return payload.insert("result", result) diff --git a/conformance/runners/python/case_runner/is_compiled_predicate.py b/conformance/runners/python/case_runner/is_compiled_predicate.py new file mode 100644 index 0000000..be83eca --- /dev/null +++ b/conformance/runners/python/case_runner/is_compiled_predicate.py @@ -0,0 +1,13 @@ +"""IsCompiledPredicate: does the case run as compiled code?""" + +from codeupipe import Payload + + +class IsCompiledPredicate: + """True when ``compiled`` is the value that was given (True unless said).""" + + def __init__(self, compiled=True): + self.compiled = compiled + + def test(self, payload: Payload) -> bool: + return payload.get("compiled") is self.compiled diff --git a/conformance/runners/python/case_runner/make_run_function_filter.py b/conformance/runners/python/case_runner/make_run_function_filter.py new file mode 100644 index 0000000..284d34e --- /dev/null +++ b/conformance/runners/python/case_runner/make_run_function_filter.py @@ -0,0 +1,33 @@ +"""MakeRunFunctionFilter: the one function that a measurement calls again and again.""" + +from codeupipe import Payload + + +class MakeRunFunctionFilter: + """Filter (sync). + + Input keys: ``case`` (a scenario), ``compiled``, ``observe``, ``hooks``, + and ``pipeline`` or ``package``. + Output keys: ``fn``: a function with no arguments that does one run of + the scenario on a new copy of its input. ``output``: what one run gives. + """ + + def call(self, payload: Payload) -> Payload: + fn = self.make(payload) + return payload.insert("fn", fn).insert("output", self.output(fn(), payload.get("compiled"))) + + def make(self, payload: Payload): + data = payload.get("case")["input"] + if not payload.get("compiled"): + pipeline = payload.get("pipeline") + return lambda: pipeline.run(Payload(dict(data))) + run = payload.get("package").run + if not payload.get("observe"): + return lambda: run(dict(data)) + hooks = payload.get("hooks") + state = {} + return lambda: run(dict(data), state=state, hooks=hooks) + + def output(self, out, compiled): + """The dynamic run gives a Payload, and compiled code gives a dict.""" + return out if compiled else out.to_dict() diff --git a/conformance/runners/python/case_runner/measure_filter.py b/conformance/runners/python/case_runner/measure_filter.py new file mode 100644 index 0000000..092fa5e --- /dev/null +++ b/conformance/runners/python/case_runner/measure_filter.py @@ -0,0 +1,61 @@ +"""MeasureFilter: how long one run takes.""" + +import statistics +import time + +from codeupipe import Payload + +WARMUP = 20000 +RUNS = 20000 +REPEATS = 5 + +_REFERENCE_KEYS = tuple("k%d" % i for i in range(10)) + + +def _reference_write(data, key): + data[key] = data.get(key, 0) + 1 + return data + + +class MeasureFilter: + """Filter (sync). + + Input keys: ``fn`` (from MakeRunFunctionFilter). + Output keys: ``us_per_run`` (µs for one run: the best of REPEATS x RUNS) + and ``relative`` (the time of ``fn`` as a ratio to a reference workload). + + The relative time is for comparing runs on different machines and under + different load: each repeat times the reference workload and then + ``fn``, back to back, so whatever slows one slows the other, and the + median of the ratios of the repeats is reported. + """ + + def call(self, payload: Payload) -> Payload: + us_per_run, relative = self.measure(payload.get("fn")) + return payload.insert("us_per_run", us_per_run).insert("relative", relative) + + def measure(self, fn): + for _ in range(WARMUP): + fn() + self.reference() + best, ratios = None, [] + for _ in range(REPEATS): + reference = self.timed(self.reference) + elapsed = self.timed(fn) + ratios.append(elapsed / reference) + best = elapsed if best is None else min(best, elapsed) + return best / RUNS * 1e6, statistics.median(ratios) + + def timed(self, fn) -> float: + started = time.perf_counter() + for _ in range(RUNS): + fn() + return time.perf_counter() - started + + @staticmethod + def reference(): + """The reference workload: ten dict writes behind ten function calls, with no framework.""" + data = {"a": 1} + for key in _REFERENCE_KEYS: + data = _reference_write(data, key) + return data diff --git a/conformance/runners/python/case_runner/name_steps_filter.py b/conformance/runners/python/case_runner/name_steps_filter.py new file mode 100644 index 0000000..5661b5a --- /dev/null +++ b/conformance/runners/python/case_runner/name_steps_filter.py @@ -0,0 +1,35 @@ +"""NameStepsFilter: give each step in the result the name that the case gives it.""" + +from codeupipe import Payload + + +class NameStepsFilter: + """Filter (sync). + + Input keys: ``result``, ``label`` ({step label: name in the case}). + Output keys: ``result``, with the name of the case for each step in + ``state``, ``pipeline_state``, ``inner_states`` and ``hooks``. A label + that is not in ``label`` stays as it is. + """ + + def call(self, payload: Payload) -> Payload: + label = payload.get("label") + named = {key: self.named(key, value, label) for key, value in payload.get("result").items()} + return payload.insert("result", named) + + def named(self, key, value, label): + """The value of one key of the result, with the names of the case where it has steps.""" + if key in ("state", "pipeline_state"): + return self.state(value, label) + if key == "inner_states": + return [self.state(one, label) for one in value] + if key == "hooks": + return [self.event(event, label) for event in value] + return value + + def state(self, state, label) -> dict: + return {key: [label.get(step, step) for step in steps] for key, steps in state.items()} + + def event(self, event, label) -> str: + phase, _, step = event.partition(":") + return f"{phase}:{label.get(step, step)}" diff --git a/conformance/runners/python/case_runner/parse_target_filter.py b/conformance/runners/python/case_runner/parse_target_filter.py new file mode 100644 index 0000000..fab6c17 --- /dev/null +++ b/conformance/runners/python/case_runner/parse_target_filter.py @@ -0,0 +1,36 @@ +"""ParseTargetFilter: what the name of a target means.""" + +from codeupipe import Payload + +_DYNAMIC = {"compiled": False, "run_async": False, "lowering": "", "observe": False} +_TARGETS = { + "run": _DYNAMIC, + "run_async": {**_DYNAMIC, "run_async": True}, + "static-steps": {"compiled": True, "run_async": False, "lowering": "steps", "observe": False}, + "static-inline": {"compiled": True, "run_async": False, "lowering": "inline", "observe": False}, + "static-observe": {"compiled": True, "run_async": False, "lowering": "steps", "observe": True}, +} + + +class ParseTargetFilter: + """Filter (sync). + + Input keys: ``target`` (``run``, ``run_async``, ``static-steps``, + ``static-inline`` or ``static-observe``). + Output keys: ``compiled`` (the case runs as compiled code), ``run_async``, + ``lowering`` (the layout of the compiled code) and ``observe`` (the + compiled code records State and calls Hooks). + + ``allowed`` is the targets that this run takes, and ``what`` is the word + for them in the message for another name. + """ + + def __init__(self, allowed=tuple(_TARGETS), what="target"): + self.allowed = allowed + self.what = what + + def call(self, payload: Payload) -> Payload: + target = payload.get("target") + if target not in self.allowed: + raise SystemExit(f"unknown {self.what} {target!r}") + return payload.merge(Payload(dict(_TARGETS[target]))) diff --git a/conformance/runners/python/case_runner/run_case_pipeline.py b/conformance/runners/python/case_runner/run_case_pipeline.py new file mode 100644 index 0000000..6b72a81 --- /dev/null +++ b/conformance/runners/python/case_runner/run_case_pipeline.py @@ -0,0 +1,36 @@ +"""RunCasePipeline: play one conformance case on one target and report what happened.""" + +from codeupipe import Pipeline + +from .attach_record_hook_filter import AttachRecordHookFilter +from .build_pipeline_filter import BuildPipelineFilter +from .export_filter import ExportFilter +from .finish_result_filter import FinishResultFilter +from .is_compiled_predicate import IsCompiledPredicate +from .name_steps_filter import NameStepsFilter +from .parse_target_filter import ParseTargetFilter +from .run_compiled_filter import RunCompiledFilter +from .run_dynamic_filter import RunDynamicFilter +from .write_predicates_filter import WritePredicatesFilter + + +class RunCasePipeline(Pipeline): + """Input keys: ``case``, ``target``, ``workdir``, ``module_name``. + Output keys: ``result`` (conformance/README.md, "Runner protocol"). + + The runner never judges. A step that raises Unsupported, or any other + error before the case runs, means that the target cannot express the + case: RunEachCaseFilter reports that. + """ + + def __init__(self): + super().__init__() + self.add_filter(ParseTargetFilter()) + self.add_filter(WritePredicatesFilter()) + self.add_filter(BuildPipelineFilter()) + self.add_filter(AttachRecordHookFilter()) + self.add_valve(IsCompiledPredicate(), ExportFilter()) + self.add_valve(IsCompiledPredicate(False), RunDynamicFilter()) + self.add_valve(IsCompiledPredicate(), RunCompiledFilter()) + self.add_filter(NameStepsFilter()) + self.add_filter(FinishResultFilter()) diff --git a/conformance/runners/python/case_runner/run_compiled_filter.py b/conformance/runners/python/case_runner/run_compiled_filter.py new file mode 100644 index 0000000..ebef4d5 --- /dev/null +++ b/conformance/runners/python/case_runner/run_compiled_filter.py @@ -0,0 +1,55 @@ +"""RunCompiledFilter: run the case as compiled code.""" + +import copy + +import conformance_sink as sink +from codeupipe import Payload + + +class RunCompiledFilter: + """Filter (sync). + + Input keys: ``case``, ``package`` (from ExportFilter), ``observe``, + ``hooks``. + Output keys: ``result``: ``output`` or ``error``, and ``input_after``. + With ``observe`` also ``state``, ``inner_states`` and, for a case that + asks for them, ``hooks``. Compiled code is a function, so it has no + ``pipeline_state``. The steps in the result are labels: NameStepsFilter + gives them the names of the case. + """ + + def call(self, payload: Payload) -> Payload: + case, package, observe = payload.get("case"), payload.get("package"), payload.get("observe") + hooks = payload.get("hooks") + data = copy.deepcopy(case["input"]) + result = {} + state = {} + inner_states = [] + + def again(inner): + # The compiled form is a function. To run the pipeline again is to call that function again, + # and the caller gives each call its own State. + if not observe: + return package.run(inner) + inner_states.append({}) + return package.run(inner, state=inner_states[-1], hooks=None) + + sink.RUN_AGAIN[0] = again + try: + if observe: + result["output"] = package.run(data, state=state, hooks=hooks) + else: + result["output"] = package.run(data) + except Exception as error: # noqa: BLE001 + result["error"] = f"{type(error).__name__}: {error}" + if observe: + result["state"] = self.report(state) + result["inner_states"] = [self.report(one) for one in inner_states] + if case.get("hooks"): + result["hooks"] = list(sink.HOOKS) + result["input_after"] = data + return payload.insert("result", result) + + def report(self, state) -> dict: + return {"executed": list(state["executed"]), "skipped": list(state["skipped"]), + "errors": [step for step, _error in state["errors"]]} diff --git a/conformance/runners/python/case_runner/run_dynamic_filter.py b/conformance/runners/python/case_runner/run_dynamic_filter.py new file mode 100644 index 0000000..8bd4d0b --- /dev/null +++ b/conformance/runners/python/case_runner/run_dynamic_filter.py @@ -0,0 +1,56 @@ +"""RunDynamicFilter: run the case with ``Pipeline.run`` or ``Pipeline.run_async``.""" + +import asyncio +import copy + +import conformance_sink as sink +from codeupipe import Payload, State + + +class RunDynamicFilter: + """Filter (sync). + + Input keys: ``case``, ``pipeline``, ``run_async``, ``hooks``. + Output keys: ``result``: ``output`` or ``error``, ``state`` (the State of + the run), ``pipeline_state`` (``state`` on the Pipeline after the run), + ``inner_states`` (the State of each run that a run_again step started), + ``hooks`` for a case that asks for them, and ``input_after``. The steps + in it are labels: NameStepsFilter gives them the names of the case. + """ + + def call(self, payload: Payload) -> Payload: + case, pipeline, run_async = payload.get("case"), payload.get("pipeline"), payload.get("run_async") + data = copy.deepcopy(case["input"]) + # With "given_state", the caller gives the State of the run. If not, the run makes one. + given = [State()] if case.get("given_state") else [] + result = {} + + def again(inner): + state = State() + sink.INNER_STATES.append(state) + return pipeline.run_async(Payload(inner), state) if run_async else pipeline.run(Payload(inner), state) + + sink.RUN_AGAIN[0] = again + try: + if run_async: + out = asyncio.run(pipeline.run_async(Payload(data), *given)) + else: + out = pipeline.run(Payload(data), *given) + result["output"] = out.to_dict() + except Exception as error: # noqa: BLE001 — the error is the result + result["error"] = f"{type(error).__name__}: {error}" + + # The State that these two lines read is that of the pipeline of the case, which this step ran. + # It is what the case asks for, not the State of the run that this step is in. + ran = pipeline.state # cup: allow-state-read the State of the pipeline of the case is the result that this step reports + result["state"] = self.report(given[0] if given else ran) + result["pipeline_state"] = self.report(ran) + result["inner_states"] = [self.report(state) for state in sink.INNER_STATES] + if payload.get("hooks"): + result["hooks"] = list(sink.HOOKS) + result["input_after"] = data + return payload.insert("result", result) + + def report(self, state) -> dict: + return {"executed": list(state.executed), "skipped": list(state.skipped), + "errors": [step for step, _error in state.errors]} diff --git a/conformance/runners/python/case_runner/run_each_case_filter.py b/conformance/runners/python/case_runner/run_each_case_filter.py new file mode 100644 index 0000000..fd3d8e0 --- /dev/null +++ b/conformance/runners/python/case_runner/run_each_case_filter.py @@ -0,0 +1,40 @@ +"""RunEachCaseFilter: play each case, one after the other, each in its own directory.""" + +import pathlib +import tempfile + +import conformance_sink as sink +from codeupipe import Payload + +from .unsupported import Unsupported + + +class RunEachCaseFilter: + """Filter (sync). + + Input keys: ``cases``, ``target``. + Output keys: ``results``: for each case its result, or ``{"id", + "unsupported": reason}`` when the target cannot express it. + + ``each`` is the pipeline that plays one case. + """ + + def __init__(self, each): + self.each = each + + def call(self, payload: Payload) -> Payload: + target = payload.get("target") + results = [self.one(case, target, index) for index, case in enumerate(payload.get("cases"))] + return payload.insert("results", results) + + def one(self, case, target, index) -> dict: + sink.reset() + workdir = pathlib.Path(tempfile.mkdtemp(prefix="cup_conf_")) + try: + out = self.each.run(Payload({"case": case, "target": target, "workdir": workdir, + "module_name": f"conf_predicates_{index}"})) + except Unsupported as reason: + return {"id": case["id"], "unsupported": str(reason)} + except Exception as error: # noqa: BLE001 — for example the compiler refuses a shape + return {"id": case["id"], "unsupported": f"{type(error).__name__}: {error}"} + return out.get("result") diff --git a/conformance/runners/python/case_runner/runner_pipeline.py b/conformance/runners/python/case_runner/runner_pipeline.py new file mode 100644 index 0000000..0ca304c --- /dev/null +++ b/conformance/runners/python/case_runner/runner_pipeline.py @@ -0,0 +1,15 @@ +"""RunnerPipeline: the whole runner for one call: each case, or each scenario, on one target.""" + +from codeupipe import Pipeline + + +class RunnerPipeline(Pipeline): + """Input keys: ``cases``, ``target``. Output keys: ``results``. + + ``each`` is the Filter that goes through the cases: RunEachCaseFilter to + play them, or BenchEachScenarioFilter to time them. + """ + + def __init__(self, each): + super().__init__() + self.add_filter(each) diff --git a/conformance/runners/python/case_runner/unsupported.py b/conformance/runners/python/case_runner/unsupported.py new file mode 100644 index 0000000..0d2cb73 --- /dev/null +++ b/conformance/runners/python/case_runner/unsupported.py @@ -0,0 +1,9 @@ +"""The errors that say "this target cannot express this case".""" + + +class Unsupported(Exception): + """The target cannot express the case. The message is the reason.""" + + +class ExportRefused(Unsupported): + """The compiler refused the pipeline of the case.""" diff --git a/conformance/runners/python/case_runner/write_predicates_filter.py b/conformance/runners/python/case_runner/write_predicates_filter.py new file mode 100644 index 0000000..09145ff --- /dev/null +++ b/conformance/runners/python/case_runner/write_predicates_filter.py @@ -0,0 +1,55 @@ +"""WritePredicatesFilter: the Predicates of a case, as real classes in a module on disk.""" + +import importlib +import json +import sys + +from codeupipe import Payload + +from .unsupported import Unsupported + + +class WritePredicatesFilter: + """Filter (sync). + + Input keys: ``case``, ``workdir`` (a directory for this case), + ``module_name``. + Output keys: ``predicates``: a module that has, for each Valve of the + case, an instance ``when_`` of a Predicate class. + + The Predicates are written as source code and imported, because the + compiler reads the source of a Predicate and refuses anything else. + """ + + def call(self, payload: Payload) -> Payload: + workdir, module_name = payload.get("workdir"), payload.get("module_name") + sources = self.sources(payload.get("case")["steps"]) + (workdir / f"{module_name}.py").write_text("\n\n".join(sources) or "\n") + sys.path.insert(0, str(workdir)) + try: + return payload.insert("predicates", importlib.import_module(module_name)) + finally: + sys.path.remove(str(workdir)) + + def sources(self, steps) -> list: + """The source of the Predicate of each Valve, with those of the nested pipelines.""" + found = [] + for step in steps: + if step["kind"] == "valve": + found.append(self.source(step["name"], step["when"])) + elif step["kind"] == "pipeline": + found.extend(self.sources(step["steps"])) + return found + + def source(self, name, when) -> str: + key = json.dumps(when["key"]) + if when["op"] == "equals": + body = f"payload.get({key}) == {when['value']!r}" + elif when["op"] == "truthy": + body = f"bool(payload.get({key}))" + else: + raise Unsupported(f"unknown valve op {when['op']!r}") + # a predicate object (conformance/API.md, "Valve"): a class with test(), + # plus a module-level instance under the name the builders look up + return (f"class When_{name}:\n def test(self, payload):\n return {body}\n\n\n" + f"when_{name} = When_{name}()\n") diff --git a/conformance/runners/python/conformance_sink.py b/conformance/runners/python/conformance_sink.py index e815e87..f9c6073 100644 --- a/conformance/runners/python/conformance_sink.py +++ b/conformance/runners/python/conformance_sink.py @@ -4,13 +4,19 @@ ``from conformance_sink import TAPS`` as an import, so every lowered step appends to this one list. A list defined next to the ops would be *copied* into each lowered step instead (see the known limitation in -packages/codeupipe/codeupipe/lower/filters/emit_inline_module_filter.py). +packages/codeupipe/codeupipe/compile/filters/emit_inline_module_filter.py). """ TAPS = [] HOOKS = [] +# The State of each run that a run_again step started, in order. +INNER_STATES = [] +# How the target runs its pipeline one more time: one function, that the runner sets for each case. +RUN_AGAIN = [None] def reset(): del TAPS[:] del HOOKS[:] + del INNER_STATES[:] + RUN_AGAIN[0] = None diff --git a/conformance/runners/python/cup_ops/__init__.py b/conformance/runners/python/cup_ops/__init__.py index b9a7b47..384beca 100644 --- a/conformance/runners/python/cup_ops/__init__.py +++ b/conformance/runners/python/cup_ops/__init__.py @@ -6,6 +6,7 @@ """ from .add_filter import AddFilter +from .async_run_again_filter import AsyncRunAgainFilter from .append_filter import AppendFilter from .copy_filter import CopyFilter from .fail_filter import FailFilter @@ -19,11 +20,13 @@ from .record_hook import RecordHook from .record_tap import RecordTap from .remove_filter import RemoveFilter +from .run_again_filter import RunAgainFilter from .set_filter import SetFilter __all__ = [ "AddFilter", "AppendFilter", + "AsyncRunAgainFilter", "CopyFilter", "FailFilter", "FailTap", @@ -36,5 +39,6 @@ "RecordHook", "RecordTap", "RemoveFilter", + "RunAgainFilter", "SetFilter", ] diff --git a/conformance/runners/python/cup_ops/async_run_again_filter.py b/conformance/runners/python/cup_ops/async_run_again_filter.py new file mode 100644 index 0000000..cfc7e98 --- /dev/null +++ b/conformance/runners/python/cup_ops/async_run_again_filter.py @@ -0,0 +1,15 @@ +"""AsyncRunAgainFilter: the reference `run_again` step for ``run_async`` (conformance/README.md).""" + +from conformance_sink import RUN_AGAIN + + +class AsyncRunAgainFilter: + """``RunAgainFilter`` for a pipeline that runs with ``run_async``.""" + + def __init__(self, data): + self.data = data + + async def call(self, payload): + if payload.get("inner") is not True: + await RUN_AGAIN[0](dict(self.data)) + return payload diff --git a/conformance/runners/python/cup_ops/nested_pipeline.py b/conformance/runners/python/cup_ops/nested_pipeline.py index ab078c3..5812f5d 100644 --- a/conformance/runners/python/cup_ops/nested_pipeline.py +++ b/conformance/runners/python/cup_ops/nested_pipeline.py @@ -5,4 +5,4 @@ class NestedPipeline(Pipeline): """The runner's named pipeline type: nested pipelines must be named - types (conformance/API.md, step identity).""" + types (conformance/API.md, step labels).""" diff --git a/conformance/runners/python/cup_ops/record_hook.py b/conformance/runners/python/cup_ops/record_hook.py index cb6216e..679266f 100644 --- a/conformance/runners/python/cup_ops/record_hook.py +++ b/conformance/runners/python/cup_ops/record_hook.py @@ -5,7 +5,7 @@ class RecordHook: """Records ``before:``, ``after:``, ``error:``; the - pipeline-level events use an empty step. Hooks receive step identities; + pipeline-level events use an empty step. Hooks receive step labels; the runner maps them back to the case's step names (runner.labels).""" def before(self, step, payload): diff --git a/conformance/runners/python/cup_ops/run_again_filter.py b/conformance/runners/python/cup_ops/run_again_filter.py new file mode 100644 index 0000000..7408b24 --- /dev/null +++ b/conformance/runners/python/cup_ops/run_again_filter.py @@ -0,0 +1,20 @@ +"""RunAgainFilter: the reference `run_again` step (conformance/README.md).""" + +from conformance_sink import RUN_AGAIN + + +class RunAgainFilter: + """Starts a second run of the pipeline that it is in, inside the first one. + + How a pipeline is run again depends on the target, so the runner gives + that as a function, in ``RUN_AGAIN``. The second run has ``inner`` in its + input, so it does not start a third one. + """ + + def __init__(self, data): + self.data = data + + def call(self, payload): + if payload.get("inner") is not True: + RUN_AGAIN[0](dict(self.data)) + return payload diff --git a/conformance/runners/python/runner.py b/conformance/runners/python/runner.py index 759395f..77d5d5a 100644 --- a/conformance/runners/python/runner.py +++ b/conformance/runners/python/runner.py @@ -5,9 +5,15 @@ Targets run codeupipe Pipeline.run (sync, the default) run_async codeupipe Pipeline.run_async (inside asyncio.run) - static-steps codeupipe.lower export, per-step layout, loaded from disk - static-inline codeupipe.lower export, lowering="inline", loaded from disk - static-observe codeupipe.lower export, observe=True: State and Hooks too + static-steps codeupipe.compile export, per-step layout, loaded from disk + static-inline codeupipe.compile export, lowering="inline", loaded from disk + static-observe codeupipe.compile export, observe=True: State and Hooks too + +With ``bench:`` it times the scenarios that it gets, and does not +play cases (conformance/bench.py). + +The runner is two codeupipe pipelines, in ``case_runner/``: one plays a +case, and one times a scenario. This file is its command line. The runner never judges; conformance/run.py compares every target's results against the same expectations. The fast static targets have no State or @@ -15,338 +21,33 @@ static-observe reports both and is judged on them. """ -import asyncio -import copy -import importlib import json import pathlib import sys -import tempfile HERE = pathlib.Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) -import conformance_sink as sink # noqa: E402 -import cup_ops as ops # noqa: E402 -from codeupipe import Payload, Pipeline # noqa: E402 - # noqa: E402 - - -class Unsupported(Exception): - pass - - -def _filter(spec): - op = spec["op"] - if op == "set": - return ops.SetFilter(spec["key"], spec["value"]) - if op == "copy": - return ops.CopyFilter(spec["from"], spec["to"]) - if op == "add": - return ops.AddFilter(spec["key"], spec["by"]) - if op == "append": - return ops.AppendFilter(spec["key"], spec["value"]) - if op == "remove": - return ops.RemoveFilter(spec["key"]) - if op == "merge": - return ops.MergeFilter(spec["value"]) - if op == "has": - return ops.HasFilter(spec["key"], spec["to"]) - if op == "mutate": - return ops.MutateFilter(spec["key"], spec["value"]) - if op == "fail": - return ops.FailFilter(spec["message"]) - raise Unsupported(f"unknown filter op {op!r}") - - -def _tap(spec): - if spec["op"] == "record": - return ops.RecordTap(spec["key"]) - if spec["op"] == "fail": - return ops.FailTap(spec["message"]) - if spec["op"] == "noop": - return ops.NoopTap() - raise Unsupported(f"unknown tap op {spec['op']!r}") - - -def _predicate_source(name, when): - """Valve predicates are generated as real predicate classes so static - lowering can recover their source (it refuses anything else).""" - key = json.dumps(when["key"]) - if when["op"] == "equals": - body = f"payload.get({key}) == {when['value']!r}" - elif when["op"] == "truthy": - body = f"bool(payload.get({key}))" - else: - raise Unsupported(f"unknown valve op {when['op']!r}") - # a predicate object (conformance/API.md, "Valve"): a class with test(), - # plus a module-level instance under the name the builders look up - return (f"class When_{name}:\n def test(self, payload):\n return {body}\n\n\n" - f"when_{name} = When_{name}()\n") - - -def _predicates(steps, module_dir, module_name): - sources = [] - - def walk(items): - for step in items: - if step["kind"] == "valve": - sources.append(_predicate_source(step["name"], step["when"])) - elif step["kind"] == "pipeline": - walk(step["steps"]) - - walk(steps) - path = module_dir / f"{module_name}.py" - path.write_text("\n\n".join(sources) or "\n") - sys.path.insert(0, str(module_dir)) - try: - return importlib.import_module(module_name) - finally: - sys.path.remove(str(module_dir)) - - -def build(case, cls, predicates, names): - p = cls() - for step in case["steps"]: - kind, name = step["kind"], step["name"] - if kind == "filter": - obj = _filter(step) - p.add_filter(obj) - elif kind == "tap": - obj = _tap(step) - p.add_tap(obj) - elif kind == "valve": - p.add_valve(getattr(predicates, f"when_{name}"), _filter(step["filter"])) - obj = p._steps[-1][1] # the Valve that the pipeline made: the step that labels() names - elif kind == "pipeline": - if not hasattr(p, "add_pipeline"): - raise Unsupported(f"{cls.__name__} has no add_pipeline") - obj = build({"steps": step["steps"]}, ops.NestedPipeline, predicates, names) - p.add_pipeline(obj) - else: - raise Unsupported(f"unknown step kind {kind!r}") - names[id(obj)] = name - return p - - -def labels(p, names): - """identity -> the case's step name, for the steps of ``p``. - - Steps are identified by their filter (conformance/API.md), while the - shared cases name them. The runner built every step, so it knows which - case step each identity belongs to. An identity reported that is not in - this map (wrong, or inconsistent between modes) stays as is and fails - the comparison. - """ - return {identity: names[id(step)] for identity, step, _type in p._steps} - - -def _named(steps, label): - return [label.get(step, step) for step in steps] - - -def _named_hooks(events, label): - out = [] - for event in events: - phase, _, step = event.partition(":") - out.append(f"{phase}:{label.get(step, step)}") - return out - - -def _dynamic(case, predicates, run_async=False): - names = {} - p = build(case, Pipeline, predicates, names) - label = labels(p, names) - if case.get("hooks"): - p.use_hook(ops.RecordHook()) - data = copy.deepcopy(case["input"]) - result = {} - try: - if run_async: - out = asyncio.run(p.run_async(Payload(data))) - else: - out = p.run(Payload(data)) - result["output"] = out.to_dict() - except Exception as error: # noqa: BLE001 — the error is the result - result["error"] = f"{type(error).__name__}: {error}" - state = p.state - result["state"] = { - "executed": _named(state.executed, label), - "skipped": _named(state.skipped, label), - "errors": _named([step for step, _ in state.errors], label), - } - if case.get("hooks"): - result["hooks"] = _named_hooks(sink.HOOKS, label) - result["input_after"] = data - return result - - -def _load_export(p, lowering, observe, workdir): - """Export ``p`` and import the package from disk, as a customer would.""" - from codeupipe.lower import build_export_pipeline_v2 - - out = build_export_pipeline_v2(write=False).run( - Payload({"pipeline": p, "pattern": "flat", "lowering": lowering, "observe": observe})) - for f in out.get("files") or []: - target = workdir / f["path"] - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(f["content"]) - package_parent = str(workdir / "src") - for name in [m for m in sys.modules if m == "steps" or m.startswith("steps.")]: - del sys.modules[name] - sys.path.insert(0, package_parent) - try: - return importlib.import_module("steps") - finally: - sys.path.remove(package_parent) - - -def _static(case, predicates, lowering, workdir, observe=False): - names = {} - p = build(case, Pipeline, predicates, names) - label = labels(p, names) - hooks = [ops.RecordHook()] if observe and case.get("hooks") else None - if hooks: - p.use_hook(hooks[0]) - package = _load_export(p, lowering, observe, workdir) - data = copy.deepcopy(case["input"]) - result = {} - state = {} - try: - if observe: - result["output"] = package.run(data, state=state, hooks=hooks) - else: - result["output"] = package.run(data) - except Exception as error: # noqa: BLE001 - result["error"] = f"{type(error).__name__}: {error}" - if observe: - result["state"] = { - "executed": _named(state["executed"], label), - "skipped": _named(state["skipped"], label), - "errors": _named([step for step, _ in state["errors"]], label), - } - if case.get("hooks"): - result["hooks"] = _named_hooks(sink.HOOKS, label) - result["input_after"] = data - return result - - -def run_case(case, target, index): - sink.reset() - tmp = pathlib.Path(tempfile.mkdtemp(prefix="cup_conf_")) - try: - predicates = _predicates(case["steps"], tmp, f"conf_predicates_{index}") - if target == "run": - result = _dynamic(case, predicates) - elif target == "run_async": - result = _dynamic(case, predicates, run_async=True) - elif target in ("static-steps", "static-inline"): - result = _static(case, predicates, target.split("-", 1)[1], tmp) - elif target == "static-observe": - result = _static(case, predicates, "steps", tmp, observe=True) - else: - raise SystemExit(f"unknown target {target!r}") - except Unsupported as reason: - return {"id": case["id"], "unsupported": str(reason)} - except Exception as error: # noqa: BLE001 — e.g. export refusing a shape - return {"id": case["id"], "unsupported": f"{type(error).__name__}: {error}"} - result["id"] = case["id"] - result["taps"] = list(sink.TAPS) - return result - - -BENCH_WARMUP = 20000 -BENCH_RUNS = 20000 -BENCH_REPEATS = 5 - - -_REFERENCE_KEYS = tuple("k%d" % i for i in range(10)) - - -def _reference_write(data, key): - data[key] = data.get(key, 0) + 1 - return data - - -def _reference(): - """The reference workload: ten dict writes behind ten function calls, - with no framework. A pipeline's time is reported relative to it.""" - data = {"a": 1} - for key in _REFERENCE_KEYS: - data = _reference_write(data, key) - return data - - -def _timed(fn): - import time - t0 = time.perf_counter() - for _ in range(BENCH_RUNS): - fn() - return time.perf_counter() - t0 - - -def _measure(fn): - """(µs per run, time relative to the reference workload). - - µs per run is the best of BENCH_REPEATS x BENCH_RUNS. The relative time - is for comparing runs on different machines and under different load: - each repeat times the reference workload and then ``fn``, back to back, - so whatever slows one slows the other, and the median of the repeats' - ratios is reported. - """ - import statistics - for _ in range(BENCH_WARMUP): - fn() - _reference() - best, ratios = None, [] - for _ in range(BENCH_REPEATS): - reference = _timed(_reference) - elapsed = _timed(fn) - ratios.append(elapsed / reference) - best = elapsed if best is None else min(best, elapsed) - return best / BENCH_RUNS * 1e6, statistics.median(ratios) +from codeupipe import Payload # noqa: E402 +from case_runner.bench_each_scenario_filter import BenchEachScenarioFilter # noqa: E402 +from case_runner.bench_scenario_pipeline import BenchScenarioPipeline # noqa: E402 +from case_runner.run_case_pipeline import RunCasePipeline # noqa: E402 +from case_runner.run_each_case_filter import RunEachCaseFilter # noqa: E402 +from case_runner.runner_pipeline import RunnerPipeline # noqa: E402 -def bench(target, scenarios): - results = [] - for index, case in enumerate(scenarios): - tmp = pathlib.Path(tempfile.mkdtemp(prefix="cup_bench_")) - predicates = _predicates(case["steps"], tmp, f"bench_predicates_{index}") - p = build(case, Pipeline, predicates, {}) - if case.get("hooks") == "noop": - p.use_hook(ops.NoopHook()) - data = case["input"] - if target == "run": - fn = lambda: p.run(Payload(dict(data))) # noqa: E731 - output = fn().to_dict() - elif target in ("static-inline", "static-steps"): - try: - run = _load_export(p, target.split("-", 1)[1], False, tmp).run - except Exception as error: # noqa: BLE001 — export refused the shape (e.g. Hooks) - results.append({"id": case["id"], "unsupported": f"{type(error).__name__}: {error}"}) - continue - fn = lambda: run(dict(data)) # noqa: E731 - output = fn() - elif target == "static-observe": - run = _load_export(p, "steps", True, tmp).run - hooks = [ops.NoopHook()] if case.get("hooks") == "noop" else None - state = {} - fn = lambda: run(dict(data), state=state, hooks=hooks) # noqa: E731 - output = fn() - else: - raise SystemExit(f"unknown bench target {target!r}") - us_per_run, relative = _measure(fn) - results.append({"id": case["id"], "us_per_run": us_per_run, "relative": relative, "output": output}) - return results +# Is the first argument "bench:"? Then the runner times scenarios. If not, it plays cases. +_PIPELINES = { + False: lambda: RunnerPipeline(RunEachCaseFilter(RunCasePipeline())), + True: lambda: RunnerPipeline(BenchEachScenarioFilter(BenchScenarioPipeline())), +} def main(): target = sys.argv[1] - if target.startswith("bench:"): - json.dump(bench(target.split(":", 1)[1], json.load(sys.stdin)), sys.stdout) - return - cases = json.load(sys.stdin) - json.dump([run_case(case, target, i) for i, case in enumerate(cases)], sys.stdout) + pipeline = _PIPELINES[target.startswith("bench:")]() + out = pipeline.run(Payload({"cases": json.load(sys.stdin), "target": target.removeprefix("bench:")})) + json.dump(out.get("results"), sys.stdout) if __name__ == "__main__": diff --git a/conformance/runners/rust/Cargo.toml b/conformance/runners/rust/Cargo.toml new file mode 100644 index 0000000..f74955c --- /dev/null +++ b/conformance/runners/rust/Cargo.toml @@ -0,0 +1,16 @@ +# The Rust conformance runner: runner < cases.json > results.json. +# It has no dependency to download, so run.py builds it with --offline. +[package] +name = "cup-conformance-rust" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +license = "Apache-2.0" +publish = false + +[[bin]] +name = "runner" +path = "src/main.rs" + +[dependencies] +codeupipe = { path = "../../../packages/codeupipe-rs" } diff --git a/conformance/runners/rust/src/bench.rs b/conformance/runners/rust/src/bench.rs new file mode 100644 index 0000000..efb6900 --- /dev/null +++ b/conformance/runners/rust/src/bench.rs @@ -0,0 +1,89 @@ +//! The bench mode: times the scenarios as the Python runner does, so that +//! conformance/bench.py can put the numbers side by side. +use std::collections::HashMap; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use codeupipe::{Map, Pipeline, Value}; + +const WARMUP: usize = 20_000; +const RUNS: usize = 20_000; +const REPEATS: usize = 5; + +const REFERENCE_KEYS: [&str; 10] = ["k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8", "k9"]; + +#[inline(never)] +fn reference_write(data: &mut HashMap, key: &str) { + *data.entry(key.to_string()).or_insert(0.0) += 1.0; +} + +/// The reference workload: ten map writes behind ten function calls, with no +/// framework. A pipeline's time is reported relative to it. +// cup: allow-plain the timing loop of the bench target: it measures the pipeline, so it is not a step of it +fn reference() -> HashMap { + let mut data = HashMap::from([("a".to_string(), 1.0)]); + for key in REFERENCE_KEYS { + reference_write(&mut data, black_box(key)); + } + data +} + +// cup: allow-plain the timing loop of the bench target: it measures the pipeline, so it is not a step of it +fn timed(f: &mut impl FnMut()) -> Duration { + let started = Instant::now(); + for _ in 0..RUNS { + f(); + } + started.elapsed() +} + +/// (µs per run, time relative to the reference workload). µs per run is the +/// best of REPEATS x RUNS. Each repeat times the reference and then `f`, +/// back to back, and the median of the ratios is reported. +// cup: allow-plain the timing loop of the bench target: it measures the pipeline, so it is not a step of it +fn measure(mut f: impl FnMut()) -> (f64, f64) { + let mut reference_run = || { + black_box(reference()); + }; + for _ in 0..WARMUP { + f(); + reference_run(); + } + let mut best = Duration::MAX; + let mut ratios = Vec::with_capacity(REPEATS); + for _ in 0..REPEATS { + let reference = timed(&mut reference_run); + let elapsed = timed(&mut f); + ratios.push(elapsed.as_secs_f64() / reference.as_secs_f64()); + best = best.min(elapsed); + } + ratios.sort_by(f64::total_cmp); + let us_per_run = best.as_secs_f64() / RUNS as f64 * 1e6; + (us_per_run, ratios[REPEATS / 2]) +} + +/// Time one scenario. `pipeline` is built from its steps; `input` is its input. +// cup: allow-plain the timing loop of the bench target: it measures the pipeline, so it is not a step of it +pub fn bench(id: Value, pipeline: Pipeline, input: codeupipe::Payload) -> Value { + let output = match pipeline.run(&input) { + Ok(output) => output.to_dict(), + Err(error) => { + return Value::Map(Map::from([ + ("id", id), + ( + "unsupported", + Value::from(format!("the scenario fails: {error}")), + ), + ])) + } + }; + let (us_per_run, relative) = measure(|| { + black_box(pipeline.run(black_box(&input)).ok()); + }); + Value::Map(Map::from([ + ("id", id), + ("us_per_run", Value::from(us_per_run)), + ("relative", Value::from(relative)), + ("output", Value::Map(output)), + ])) +} diff --git a/conformance/runners/rust/src/cup_ops/add_filter.rs b/conformance/runners/rust/src/cup_ops/add_filter.rs new file mode 100644 index 0000000..15abc46 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/add_filter.rs @@ -0,0 +1,19 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = get(key, 0) + by +pub struct AddFilter { + pub key: String, + pub by: f64, +} + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let n = match payload.get(&self.key) { + None => 0.0, + Some(Value::Number(n)) => *n, + Some(other) => return Err(format!("'{}' is not a number: {other:?}", self.key).into()), + }; + payload.insert(self.key.as_str(), n + self.by); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/append_filter.rs b/conformance/runners/rust/src/cup_ops/append_filter.rs new file mode 100644 index 0000000..9898c93 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/append_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = a copy of get(key), or of an empty list, with value added +pub struct AppendFilter { + pub key: String, + pub value: Value, +} + +impl Filter for AppendFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let mut list = match payload.get(&self.key) { + None | Some(Value::Null) => Vec::new(), + Some(Value::List(items)) => items.clone(), + Some(other) => return Err(format!("'{}' is not a list: {other:?}", self.key).into()), + }; + list.push(self.value.clone()); + payload.insert(self.key.as_str(), list); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/copy_filter.rs b/conformance/runners/rust/src/cup_ops/copy_filter.rs new file mode 100644 index 0000000..468fb8e --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/copy_filter.rs @@ -0,0 +1,15 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert to = get(from), null if missing +pub struct CopyFilter { + pub from: String, + pub to: String, +} + +impl Filter for CopyFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let value = payload.get(&self.from).cloned().unwrap_or(Value::Null); + payload.insert(self.to.as_str(), value); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/equals_predicate.rs b/conformance/runners/rust/src/cup_ops/equals_predicate.rs new file mode 100644 index 0000000..0696963 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/equals_predicate.rs @@ -0,0 +1,13 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// get(key) == value, by content; a missing key is null +pub struct EqualsPredicate { + pub key: String, + pub value: Value, +} + +impl Predicate for EqualsPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get(&self.key).unwrap_or(&Value::Null) == &self.value + } +} diff --git a/conformance/runners/rust/src/cup_ops/fail_filter.rs b/conformance/runners/rust/src/cup_ops/fail_filter.rs new file mode 100644 index 0000000..ec2ab08 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/fail_filter.rs @@ -0,0 +1,12 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// return an error that carries message +pub struct FailFilter { + pub message: String, +} + +impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err(self.message.clone().into()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/fail_tap.rs b/conformance/runners/rust/src/cup_ops/fail_tap.rs new file mode 100644 index 0000000..9f9d919 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/fail_tap.rs @@ -0,0 +1,12 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// return an error +pub struct FailTap { + pub message: String, +} + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err(self.message.clone().into()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/has_filter.rs b/conformance/runners/rust/src/cup_ops/has_filter.rs new file mode 100644 index 0000000..51d547f --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/has_filter.rs @@ -0,0 +1,15 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// insert to = has(key) +pub struct HasFilter { + pub key: String, + pub to: String, +} + +impl Filter for HasFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let has = payload.has(&self.key); + payload.insert(self.to.as_str(), has); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/merge_filter.rs b/conformance/runners/rust/src/cup_ops/merge_filter.rs new file mode 100644 index 0000000..16ae75c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/merge_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, Map, Payload, StepError}; + +/// merge(Payload(value)) +pub struct MergeFilter { + pub value: Map, +} + +impl Filter for MergeFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.merge(&Payload::from_map(self.value.clone())); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/mod.rs b/conformance/runners/rust/src/cup_ops/mod.rs new file mode 100644 index 0000000..cf2ac18 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/mod.rs @@ -0,0 +1,40 @@ +//! The steps of the cases: one type for each step, with the names that the +//! TypeScript runner uses, so that the labels are the same. +mod add_filter; +mod append_filter; +mod copy_filter; +mod equals_predicate; +mod fail_filter; +mod fail_tap; +mod has_filter; +mod merge_filter; +mod mutate_filter; +mod nested_pipeline; +mod noop_hook; +mod noop_tap; +mod record_hook; +mod record_tap; +mod remove_filter; +mod run_again_filter; +mod set_filter; +pub mod sink; +mod truthy_predicate; + +pub use add_filter::AddFilter; +pub use append_filter::AppendFilter; +pub use copy_filter::CopyFilter; +pub use equals_predicate::EqualsPredicate; +pub use fail_filter::FailFilter; +pub use fail_tap::FailTap; +pub use has_filter::HasFilter; +pub use merge_filter::MergeFilter; +pub use mutate_filter::MutateFilter; +pub use nested_pipeline::NestedPipeline; +pub use noop_hook::NoopHook; +pub use noop_tap::NoopTap; +pub use record_hook::RecordHook; +pub use record_tap::RecordTap; +pub use remove_filter::RemoveFilter; +pub use run_again_filter::RunAgainFilter; +pub use set_filter::SetFilter; +pub use truthy_predicate::TruthyPredicate; diff --git a/conformance/runners/rust/src/cup_ops/mutate_filter.rs b/conformance/runners/rust/src/cup_ops/mutate_filter.rs new file mode 100644 index 0000000..ffd1686 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/mutate_filter.rs @@ -0,0 +1,16 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert(key, value) as a statement of its own. A Rust Filter always +/// changes the Payload in place, so this is the same as SetFilter; the case +/// is kept so that every runtime runs it. +pub struct MutateFilter { + pub key: String, + pub value: Value, +} + +impl Filter for MutateFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert(self.key.as_str(), self.value.clone()); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/nested_pipeline.rs b/conformance/runners/rust/src/cup_ops/nested_pipeline.rs new file mode 100644 index 0000000..94da431 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/nested_pipeline.rs @@ -0,0 +1,9 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +/// A named pipeline type, for a pipeline that is a step of another. The +/// runner adds the steps of the case after it is made. +pub struct NestedPipeline; + +impl NamedPipeline for NestedPipeline { + fn add_steps(&self, _pipeline: &mut Pipeline) {} +} diff --git a/conformance/runners/rust/src/cup_ops/noop_hook.rs b/conformance/runners/rust/src/cup_ops/noop_hook.rs new file mode 100644 index 0000000..8b89641 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/noop_hook.rs @@ -0,0 +1,6 @@ +use codeupipe::Hook; + +/// A Hook that does nothing: the cost of having one. +pub struct NoopHook; + +impl Hook for NoopHook {} diff --git a/conformance/runners/rust/src/cup_ops/noop_tap.rs b/conformance/runners/rust/src/cup_ops/noop_tap.rs new file mode 100644 index 0000000..ef79509 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/noop_tap.rs @@ -0,0 +1,10 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// do nothing +pub struct NoopTap; + +impl Tap for NoopTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/record_hook.rs b/conformance/runners/rust/src/cup_ops/record_hook.rs new file mode 100644 index 0000000..f470f3c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/record_hook.rs @@ -0,0 +1,32 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Hook, Payload, StepError}; + +/// Records before:, after: and error:. Events of the +/// pipeline itself have an empty step. +pub struct RecordHook { + pub events: Rc>>, +} + +impl RecordHook { + fn record(&self, event: &str, step: Option<&str>) { + self.events + .borrow_mut() + .push(format!("{event}:{}", step.unwrap_or(""))); + } +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + self.record("before", step); + } + + fn after(&self, step: Option<&str>, _payload: &Payload) { + self.record("after", step); + } + + fn on_error(&self, step: Option<&str>, _error: &StepError, _payload: &Payload) { + self.record("error", step); + } +} diff --git a/conformance/runners/rust/src/cup_ops/record_tap.rs b/conformance/runners/rust/src/cup_ops/record_tap.rs new file mode 100644 index 0000000..692be4c --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/record_tap.rs @@ -0,0 +1,15 @@ +use codeupipe::{Payload, StepError, Tap, Value}; + +use super::sink; + +/// append get(key) to the runner's tap log +pub struct RecordTap { + pub key: String, +} + +impl Tap for RecordTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + sink::record(payload.get(&self.key).cloned().unwrap_or(Value::Null)); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/remove_filter.rs b/conformance/runners/rust/src/cup_ops/remove_filter.rs new file mode 100644 index 0000000..6678655 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/remove_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// remove(key) +pub struct RemoveFilter { + pub key: String, +} + +impl Filter for RemoveFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.remove(&self.key); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/run_again_filter.rs b/conformance/runners/rust/src/cup_ops/run_again_filter.rs new file mode 100644 index 0000000..b149719 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/run_again_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Map, Payload, StepError, Value}; + +use super::sink; + +/// when get("inner") is not true: run the pipeline that this step is in one +/// more time, with `input` and a new State. How a pipeline is run again +/// depends on the target, so the runner gives that as a function +/// (`sink::run_again`). Then the payload goes on with no change. +pub struct RunAgainFilter { + pub input: Map, +} + +impl Filter for RunAgainFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("inner") != Some(&Value::Bool(true)) { + sink::run_again(self.input.clone())?; + } + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/set_filter.rs b/conformance/runners/rust/src/cup_ops/set_filter.rs new file mode 100644 index 0000000..f75b1d0 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/set_filter.rs @@ -0,0 +1,14 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// insert key = value +pub struct SetFilter { + pub key: String, + pub value: Value, +} + +impl Filter for SetFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert(self.key.as_str(), self.value.clone()); + Ok(()) + } +} diff --git a/conformance/runners/rust/src/cup_ops/sink.rs b/conformance/runners/rust/src/cup_ops/sink.rs new file mode 100644 index 0000000..56c4273 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/sink.rs @@ -0,0 +1,50 @@ +//! Where the steps of the runner record what they see, and the hook for +//! running the pipeline of a case again. The runner empties it before each +//! case. +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Map, State, StepError, Value}; + +/// Runs the pipeline of the case again, with this input and a new State. +pub type RunAgain = Rc Result<(), StepError>>; + +thread_local! { + static TAPS: RefCell> = const { RefCell::new(Vec::new()) }; + static RUN_AGAIN: RefCell> = const { RefCell::new(None) }; + static INNER_STATES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +pub fn record(value: Value) { + TAPS.with(|taps| taps.borrow_mut().push(value)); +} + +/// What was recorded since the last call, and an empty log. +pub fn take() -> Vec { + TAPS.with(|taps| taps.take()) +} + +/// How the `run_again` step of this case runs its pipeline again. +pub fn set_run_again(run_again: Option) { + RUN_AGAIN.with(|slot| slot.replace(run_again)); +} + +// cup: allow-plain the edge of the runner: it runs the pipeline again for the run_again op +pub fn run_again(input: Map) -> Result<(), StepError> { + // Taken out of the cell first: the run that it starts reads the cell too. + let run_again = RUN_AGAIN.with(|slot| slot.borrow().clone()); + match run_again { + Some(run_again) => run_again(input), + None => Err("no pipeline to run again".into()), + } +} + +/// Keep the State of a run that a `run_again` step started. +pub fn keep_inner_state(state: State) { + INNER_STATES.with(|states| states.borrow_mut().push(state)); +} + +/// The States kept since the last call, and an empty list. +pub fn take_inner_states() -> Vec { + INNER_STATES.with(|states| states.take()) +} diff --git a/conformance/runners/rust/src/cup_ops/truthy_predicate.rs b/conformance/runners/rust/src/cup_ops/truthy_predicate.rs new file mode 100644 index 0000000..3ac1275 --- /dev/null +++ b/conformance/runners/rust/src/cup_ops/truthy_predicate.rs @@ -0,0 +1,20 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// get(key) is truthy, as in Python: null, false, 0, "" and an empty list +/// or map are false. A missing key is false. +pub struct TruthyPredicate { + pub key: String, +} + +impl Predicate for TruthyPredicate { + fn test(&self, payload: &Payload) -> bool { + match payload.get(&self.key) { + None | Some(Value::Null) => false, + Some(Value::Bool(b)) => *b, + Some(Value::Number(n)) => *n != 0.0, + Some(Value::String(s)) => !s.is_empty(), + Some(Value::List(items)) => !items.is_empty(), + Some(Value::Map(map)) => !map.is_empty(), + } + } +} diff --git a/conformance/runners/rust/src/json.rs b/conformance/runners/rust/src/json.rs new file mode 100644 index 0000000..4ff8e5d --- /dev/null +++ b/conformance/runners/rust/src/json.rs @@ -0,0 +1,341 @@ +//! JSON text to a codeupipe `Value` and back. A runner must not download +//! anything, so the runner has its own small reader and writer. + +use codeupipe::{Map, Value}; + +/// Read one JSON value from `text`. +// cup: allow-plain the JSON reader and writer of the runner, which must not download a crate +pub fn read(text: &str) -> Result { + let mut reader = Reader { text, at: 0 }; + let value = reader.value()?; + reader.space(); + if reader.at != text.len() { + return Err(reader.problem("text after the value")); + } + Ok(value) +} + +/// Write `value` as JSON text. +pub fn write(value: &Value) -> String { + let mut out = String::new(); + write_into(value, &mut out); + out +} + +// cup: allow-plain the JSON reader and writer of the runner, which must not download a crate +fn write_into(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + // Display gives 12 for 12.0. JSON has no NaN or infinity: null, as JavaScript writes them. + Value::Number(n) if n.is_finite() => out.push_str(&n.to_string()), + Value::Number(_) => out.push_str("null"), + Value::String(s) => write_string(s, out), + Value::List(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(','); + } + write_into(item, out); + } + out.push(']'); + } + Value::Map(map) => { + out.push('{'); + for (index, (key, item)) in map.iter().enumerate() { + if index > 0 { + out.push(','); + } + write_string(key, out); + out.push(':'); + write_into(item, out); + } + out.push('}'); + } + } +} + +// cup: allow-plain the JSON reader and writer of the runner, which must not download a crate +fn write_string(s: &str, out: &mut String) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); +} + +struct Reader<'a> { + text: &'a str, + at: usize, +} + +// cup: allow-plain the JSON reader and writer of the runner, which must not download a crate +impl Reader<'_> { + fn problem(&self, what: &str) -> String { + format!("not JSON at byte {}: {what}", self.at) + } + + fn peek(&self) -> Option { + self.text.as_bytes().get(self.at).copied() + } + + fn space(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + + fn expect(&mut self, byte: u8) -> Result<(), String> { + self.space(); + if self.peek() == Some(byte) { + self.at += 1; + Ok(()) + } else { + Err(self.problem(&format!("expected '{}'", byte as char))) + } + } + + fn word(&mut self, word: &str, value: Value) -> Result { + if self.text[self.at..].starts_with(word) { + self.at += word.len(); + Ok(value) + } else { + Err(self.problem("unknown word")) + } + } + + fn value(&mut self) -> Result { + self.space(); + match self.peek() { + Some(b'n') => self.word("null", Value::Null), + Some(b't') => self.word("true", Value::Bool(true)), + Some(b'f') => self.word("false", Value::Bool(false)), + Some(b'"') => self.string().map(Value::String), + Some(b'[') => self.list(), + Some(b'{') => self.map(), + Some(b'-' | b'0'..=b'9') => self.number(), + Some(_) => Err(self.problem("unexpected character")), + None => Err(self.problem("end of the text")), + } + } + + fn number(&mut self) -> Result { + let start = self.at; + while matches!( + self.peek(), + Some(b'-' | b'+' | b'.' | b'e' | b'E' | b'0'..=b'9') + ) { + self.at += 1; + } + self.text[start..self.at] + .parse::() + .map(Value::Number) + .map_err(|_| self.problem("bad number")) + } + + fn list(&mut self) -> Result { + self.expect(b'[')?; + let mut items = Vec::new(); + self.space(); + if self.peek() == Some(b']') { + self.at += 1; + return Ok(Value::List(items)); + } + loop { + items.push(self.value()?); + self.space(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b']') => { + self.at += 1; + return Ok(Value::List(items)); + } + _ => return Err(self.problem("expected ',' or ']'")), + } + } + } + + fn map(&mut self) -> Result { + self.expect(b'{')?; + let mut map = Map::new(); + self.space(); + if self.peek() == Some(b'}') { + self.at += 1; + return Ok(Value::Map(map)); + } + loop { + self.space(); + if self.peek() != Some(b'"') { + return Err(self.problem("expected a key")); + } + let key = self.string()?; + self.expect(b':')?; + let value = self.value()?; + map.insert(key, value); + self.space(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b'}') => { + self.at += 1; + return Ok(Value::Map(map)); + } + _ => return Err(self.problem("expected ',' or '}'")), + } + } + } + + fn string(&mut self) -> Result { + self.at += 1; // the opening quote + let mut out = String::new(); + loop { + let rest = &self.text[self.at..]; + let Some(c) = rest.chars().next() else { + return Err(self.problem("string with no end")); + }; + self.at += c.len_utf8(); + match c { + '"' => return Ok(out), + '\\' => out.push(self.escape()?), + c => out.push(c), + } + } + } + + fn escape(&mut self) -> Result { + let Some(byte) = self.peek() else { + return Err(self.problem("string with no end")); + }; + self.at += 1; + Ok(match byte { + b'"' => '"', + b'\\' => '\\', + b'/' => '/', + b'b' => '\u{8}', + b'f' => '\u{c}', + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'u' => { + let first = self.hex4()?; + if (0xd800..0xdc00).contains(&first) { + // A pair of UTF-16 halves: 😀 + if !self.text[self.at..].starts_with("\\u") { + return Err(self.problem("half of a UTF-16 pair")); + } + self.at += 2; + let second = self.hex4()?; + let code = 0x10000 + ((first - 0xd800) << 10) + (second.wrapping_sub(0xdc00)); + char::from_u32(code).ok_or_else(|| self.problem("bad UTF-16 pair"))? + } else { + char::from_u32(first).ok_or_else(|| self.problem("bad \\u escape"))? + } + } + _ => return Err(self.problem("unknown escape")), + }) + } + + fn hex4(&mut self) -> Result { + let digits = self + .text + .get(self.at..self.at + 4) + .ok_or_else(|| self.problem("short \\u escape"))?; + let code = u32::from_str_radix(digits, 16).map_err(|_| self.problem("bad \\u escape"))?; + self.at += 4; + Ok(code) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use codeupipe::Map; + + #[test] + fn reads_each_kind_of_value() { + assert_eq!(read("null").unwrap(), Value::Null); + assert_eq!(read("true").unwrap(), Value::Bool(true)); + assert_eq!(read("false").unwrap(), Value::Bool(false)); + assert_eq!(read("12").unwrap(), Value::Number(12.0)); + assert_eq!(read("-1.5e2").unwrap(), Value::Number(-150.0)); + assert_eq!(read("\"x\"").unwrap(), Value::from("x")); + assert_eq!( + read(" [1, \"two\", [3]] ").unwrap(), + Value::List(vec![ + Value::from(1), + Value::from("two"), + Value::List(vec![Value::from(3)]) + ]) + ); + assert_eq!( + read("{\"a\": 1, \"b\": {}}").unwrap(), + Value::Map(Map::from([ + ("a", Value::from(1)), + ("b", Value::Map(Map::new())) + ])) + ); + } + + #[test] + fn reads_the_escapes_of_a_string() { + assert_eq!( + read(r#""q\" b\\ s\/ n\n t\t ué pair😀""#).unwrap(), + Value::from("q\" b\\ s/ n\n t\t u\u{e9} pair\u{1f600}") + ); + } + + #[test] + fn keeps_the_order_of_the_keys() { + let value = read("{\"z\": 1, \"a\": 2, \"m\": 3}").unwrap(); + let keys: Vec<&str> = value.as_map().unwrap().keys().collect(); + assert_eq!(keys, ["z", "a", "m"]); + } + + #[test] + fn refuses_text_that_is_not_json() { + for text in ["", "[1,", "{\"a\" 1}", "tru", "\"open", "1 2", "{1: 2}"] { + assert!(read(text).is_err(), "{text:?}"); + } + } + + #[test] + fn writes_each_kind_of_value() { + let value = Value::Map(Map::from([ + ("n", Value::from(1.5)), + ("w", Value::from(12)), + ("s", Value::from("x")), + ("b", Value::from(false)), + ("z", Value::Null), + ( + "l", + Value::List(vec![Value::from(1), Value::List(vec![Value::from(3)])]), + ), + ])); + assert_eq!( + write(&value), + r#"{"n":1.5,"w":12,"s":"x","b":false,"z":null,"l":[1,[3]]}"# + ); + } + + #[test] + fn writes_the_escapes_of_a_string() { + assert_eq!( + write(&Value::from("q\" b\\ n\n c\u{1} \u{e9}")), + r#""q\" b\\ n\n c\u0001 é""# + ); + } + + #[test] + fn what_it_writes_reads_back_the_same() { + let text = r#"{"v":{"n":1.5,"s":"x","b":false,"z":null,"l":[1,"two",[3]]},"e":"a\"b"}"#; + assert_eq!(write(&read(text).unwrap()), text); + } +} diff --git a/conformance/runners/rust/src/main.rs b/conformance/runners/rust/src/main.rs new file mode 100644 index 0000000..e2e975c --- /dev/null +++ b/conformance/runners/rust/src/main.rs @@ -0,0 +1,359 @@ +//! Rust conformance runner: runs the cases and reports what happened. +//! +//! runner < cases.json > results.json +//! +//! Targets +//! run the Pipeline of packages/codeupipe-rs +//! bench:run the same, timed on the scenarios of conformance/bench +//! +//! The runner never judges; conformance/run.py compares the results of every +//! target with the same expectations. +mod bench; +mod cup_ops; +mod json; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::io::{self, Read, Write}; +use std::process::ExitCode; +use std::rc::Rc; + +use codeupipe::{Filter, Map, Payload, Pipeline, Predicate, State, Value}; +use cup_ops::{ + sink, AddFilter, AppendFilter, CopyFilter, EqualsPredicate, FailFilter, FailTap, HasFilter, + MergeFilter, MutateFilter, NestedPipeline, NoopHook, NoopTap, RecordHook, RecordTap, + RemoveFilter, RunAgainFilter, SetFilter, TruthyPredicate, +}; + +/// A case that this runner cannot express. It is reported, never judged. +struct Unsupported(String); + +fn field<'a>(spec: &'a Map, name: &str) -> Result<&'a Value, Unsupported> { + spec.get(name) + .ok_or_else(|| Unsupported(format!("the step has no '{name}'"))) +} + +fn text(spec: &Map, name: &str) -> Result { + field(spec, name)? + .as_str() + .map(str::to_string) + .ok_or_else(|| Unsupported(format!("'{name}' is not a string"))) +} + +fn number(spec: &Map, name: &str) -> Result { + field(spec, name)? + .as_f64() + .ok_or_else(|| Unsupported(format!("'{name}' is not a number"))) +} + +fn map<'a>(spec: &'a Map, name: &str) -> Result<&'a Map, Unsupported> { + field(spec, name)? + .as_map() + .ok_or_else(|| Unsupported(format!("'{name}' is not an object"))) +} + +fn value(spec: &Map, name: &str) -> Result { + field(spec, name).cloned() +} + +/// Where a Filter goes: alone, or behind a Predicate. `add_filter` and +/// `add_valve` take the type of the step, which gives its label, so the +/// Filter is handed over by its own type and not as a `Box`. +trait UseFilter { + fn add(self, filter: F); +} + +struct AsFilter<'a>(&'a mut Pipeline); + +impl UseFilter for AsFilter<'_> { + fn add(self, filter: F) { + self.0.add_filter(filter); + } +} + +struct AsValve<'a, P>(&'a mut Pipeline, P); + +impl UseFilter for AsValve<'_, P> { + fn add(self, filter: F) { + self.0.add_valve(self.1, filter); + } +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn add_filter(spec: &Map, to: impl UseFilter) -> Result<(), Unsupported> { + match text(spec, "op")?.as_str() { + "set" => to.add(SetFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "copy" => to.add(CopyFilter { + from: text(spec, "from")?, + to: text(spec, "to")?, + }), + "add" => to.add(AddFilter { + key: text(spec, "key")?, + by: number(spec, "by")?, + }), + "append" => to.add(AppendFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "remove" => to.add(RemoveFilter { + key: text(spec, "key")?, + }), + "merge" => to.add(MergeFilter { + value: map(spec, "value")?.clone(), + }), + "has" => to.add(HasFilter { + key: text(spec, "key")?, + to: text(spec, "to")?, + }), + "mutate" => to.add(MutateFilter { + key: text(spec, "key")?, + value: value(spec, "value")?, + }), + "fail" => to.add(FailFilter { + message: text(spec, "message")?, + }), + "run_again" => to.add(RunAgainFilter { + input: map(spec, "input")?.clone(), + }), + op => return Err(Unsupported(format!("unknown filter op '{op}'"))), + } + Ok(()) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn add_tap(spec: &Map, pipeline: &mut Pipeline) -> Result<(), Unsupported> { + match text(spec, "op")?.as_str() { + "record" => pipeline.add_tap(RecordTap { + key: text(spec, "key")?, + }), + "fail" => pipeline.add_tap(FailTap { + message: text(spec, "message")?, + }), + "noop" => pipeline.add_tap(NoopTap), + op => return Err(Unsupported(format!("unknown tap op '{op}'"))), + } + Ok(()) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn add_valve(when: &Map, filter: &Map, pipeline: &mut Pipeline) -> Result<(), Unsupported> { + match text(when, "op")?.as_str() { + "equals" => { + let predicate = EqualsPredicate { + key: text(when, "key")?, + value: value(when, "value")?, + }; + add_filter(filter, AsValve(pipeline, predicate)) + } + "truthy" => { + let predicate = TruthyPredicate { + key: text(when, "key")?, + }; + add_filter(filter, AsValve(pipeline, predicate)) + } + op => Err(Unsupported(format!("unknown valve op '{op}'"))), + } +} + +fn steps_of(spec: &Map) -> Result<&[Value], Unsupported> { + field(spec, "steps")? + .as_list() + .ok_or_else(|| Unsupported("'steps' is not a list".to_string())) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn build(steps: &[Value], pipeline: &mut Pipeline) -> Result<(), Unsupported> { + for step in steps { + let step = step + .as_map() + .ok_or_else(|| Unsupported("a step is not an object".to_string()))?; + match text(step, "kind")?.as_str() { + "filter" => add_filter(step, AsFilter(pipeline))?, + "tap" => add_tap(step, pipeline)?, + "valve" => add_valve(map(step, "when")?, map(step, "filter")?, pipeline)?, + "pipeline" => { + let mut inner = Pipeline::named(NestedPipeline); + build(steps_of(step)?, &mut inner)?; + pipeline.add_pipeline(inner); + } + kind => return Err(Unsupported(format!("unknown step kind '{kind}'"))), + } + } + Ok(()) +} + +/// Label to the step name of the case. The cases name their steps, and the +/// runtime reports labels (conformance/API.md). The steps are added in the +/// order of the case, so the two lists are side by side. A label that the +/// runtime reports and that is not in this map stays as it is, and fails the +/// comparison. +fn names_by_label(pipeline: &Pipeline, steps: &[Value]) -> HashMap { + pipeline + .labels() + .into_iter() + .zip(steps) + .map(|(label, step)| { + let name = step.as_map().and_then(|step| step.get("name")); + ( + label, + name.and_then(Value::as_str).unwrap_or("").to_string(), + ) + }) + .collect() +} + +fn names(labels: &[String], named: &impl Fn(&str) -> String) -> Value { + Value::List( + labels + .iter() + .map(|label| Value::from(named(label))) + .collect(), + ) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn run_case(spec: &Map) -> Result { + let steps = steps_of(spec)?; + let mut pipeline = Pipeline::new(); + build(steps, &mut pipeline)?; + let by_label = names_by_label(&pipeline, steps); + let named = |label: &str| { + by_label + .get(label) + .cloned() + .unwrap_or_else(|| label.to_string()) + }; + let hooks = spec.get("hooks") == Some(&Value::Bool(true)); + let events = Rc::new(RefCell::new(Vec::new())); + if hooks { + pipeline.use_hook(RecordHook { + events: Rc::clone(&events), + }); + } + + // A run_again step runs this pipeline again, as the TypeScript runner does. + let pipeline = Rc::new(pipeline); + let again = Rc::downgrade(&pipeline); + sink::set_run_again(Some(Rc::new(move |input: Map| { + let pipeline = again.upgrade().ok_or("the pipeline is gone")?; + let mut state = State::new(); + let result = pipeline.run_with_state(&Payload::from_map(input), &mut state); + sink::keep_inner_state(state); + result.map(|_| ()) + }))); + + let input = Payload::from_map(map(spec, "input")?.clone()); + let mut result = Map::new(); + result.insert("id", value(spec, "id")?); + // With "given_state", the caller gives the State of the run. If not, the run makes one. + let mut given = (spec.get("given_state") == Some(&Value::Bool(true))).then(State::new); + let ran = match given.as_mut() { + Some(state) => pipeline.run_with_state(&input, state), + None => pipeline.run(&input), + }; + match ran { + Ok(output) => result.insert("output", output.to_dict()), + Err(error) => result.insert("error", error.to_string()), + }; + sink::set_run_again(None); + let report = |state: &State| { + let errors: Vec = state + .errors() + .iter() + .map(|(label, _)| label.clone()) + .collect(); + Value::Map(Map::from([ + ("executed", names(state.executed(), &named)), + ("skipped", names(state.skipped(), &named)), + ("errors", names(&errors, &named)), + ])) + }; + let last = pipeline.state(); + result.insert("state", report(given.as_ref().unwrap_or(&last))); + result.insert("pipeline_state", report(&last)); + let inner: Vec = sink::take_inner_states().iter().map(report).collect(); + result.insert("inner_states", Value::List(inner)); + result.insert("taps", sink::take()); + if hooks { + let events: Vec = events + .borrow() + .iter() + .map(|event| { + let (kind, label) = event.split_once(':').unwrap_or((event, "")); + let step = if label.is_empty() { + String::new() + } else { + named(label) + }; + Value::from(format!("{kind}:{step}")) + }) + .collect(); + result.insert("hooks", Value::List(events)); + } + result.insert("input_after", input.to_dict()); + Ok(Value::Map(result)) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn bench_case(spec: &Map) -> Result { + let mut pipeline = Pipeline::new(); + build(steps_of(spec)?, &mut pipeline)?; + if spec.get("hooks") == Some(&Value::from("noop")) { + pipeline.use_hook(NoopHook); + } + let input = Payload::from_map(map(spec, "input")?.clone()); + Ok(bench::bench(value(spec, "id")?, pipeline, input)) +} + +// cup: allow-plain the entry of the runner: it builds each case's pipeline from the JSON vocabulary and reports the result +fn main() -> ExitCode { + let target = std::env::args().nth(1).unwrap_or_default(); + let run: fn(&Map) -> Result = match target.as_str() { + "run" => run_case, + "bench:run" => bench_case, + _ => { + eprintln!("unknown target '{target}'. The targets are: run, bench:run"); + return ExitCode::from(2); + } + }; + let mut text = String::new(); + if let Err(error) = io::stdin().read_to_string(&mut text) { + eprintln!("cannot read the cases: {error}"); + return ExitCode::from(2); + } + let cases = match json::read(&text) { + Ok(Value::List(cases)) => cases, + Ok(_) => { + eprintln!("the cases are not a list"); + return ExitCode::from(2); + } + Err(error) => { + eprintln!("cannot read the cases: {error}"); + return ExitCode::from(2); + } + }; + + let mut results = Vec::new(); + for case in &cases { + sink::take(); + let Some(spec) = case.as_map() else { + eprintln!("a case is not an object"); + return ExitCode::from(2); + }; + results.push(match run(spec) { + Ok(result) => result, + Err(Unsupported(reason)) => Value::Map(Map::from([ + ("id", spec.get("id").cloned().unwrap_or(Value::Null)), + ("unsupported", Value::from(reason)), + ])), + }); + } + let mut out = io::stdout().lock(); + if writeln!(out, "{}", json::write(&Value::List(results))).is_err() { + return ExitCode::from(2); + } + ExitCode::SUCCESS +} diff --git a/conformance/runners/typescript/cup-ops/add-filter.ts b/conformance/runners/typescript/cup-ops/add-filter.ts new file mode 100644 index 0000000..7e9ec65 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/add-filter.ts @@ -0,0 +1,16 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert key = get(key, 0) + by */ +export class AddFilter implements Filter { + readonly key: string; + readonly by: number; + + constructor(key: string, by: number) { + this.key = key; + this.by = by; + } + + call(payload: Payload): Payload { + return payload.insert(this.key, payload.get(this.key, 0) + this.by); + } +} diff --git a/conformance/runners/typescript/cup-ops/append-filter.ts b/conformance/runners/typescript/cup-ops/append-filter.ts new file mode 100644 index 0000000..0bdd068 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/append-filter.ts @@ -0,0 +1,17 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert key = a copy of get(key), or of an empty list, with value added */ +export class AppendFilter implements Filter { + readonly key: string; + readonly value: unknown; + + constructor(key: string, value: unknown) { + this.key = key; + this.value = value; + } + + call(payload: Payload): Payload { + const list = payload.get(this.key) ?? []; + return payload.insert(this.key, [...list, this.value]); + } +} diff --git a/conformance/runners/typescript/cup-ops/copy-filter.ts b/conformance/runners/typescript/cup-ops/copy-filter.ts new file mode 100644 index 0000000..b0426db --- /dev/null +++ b/conformance/runners/typescript/cup-ops/copy-filter.ts @@ -0,0 +1,16 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert to = get(from), null if missing */ +export class CopyFilter implements Filter { + readonly from: string; + readonly to: string; + + constructor(from: string, to: string) { + this.from = from; + this.to = to; + } + + call(payload: Payload): Payload { + return payload.insert(this.to, payload.get(this.from)); + } +} diff --git a/conformance/runners/typescript/cup-ops/equals-predicate.ts b/conformance/runners/typescript/cup-ops/equals-predicate.ts new file mode 100644 index 0000000..81e018b --- /dev/null +++ b/conformance/runners/typescript/cup-ops/equals-predicate.ts @@ -0,0 +1,16 @@ +import type { Payload, Predicate } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** get(key) == value */ +export class EqualsPredicate implements Predicate { + readonly key: string; + readonly value: unknown; + + constructor(key: string, value: unknown) { + this.key = key; + this.value = value; + } + + test(payload: Payload): boolean { + return payload.get(this.key) === this.value; + } +} diff --git a/conformance/runners/typescript/cup-ops/fail-filter.ts b/conformance/runners/typescript/cup-ops/fail-filter.ts new file mode 100644 index 0000000..29f5cf4 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/fail-filter.ts @@ -0,0 +1,14 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** throw an error that carries message */ +export class FailFilter implements Filter { + readonly message: string; + + constructor(message: string) { + this.message = message; + } + + call(_payload: Payload): Payload { + throw new Error(this.message); + } +} diff --git a/conformance/runners/typescript/cup-ops/fail-tap.ts b/conformance/runners/typescript/cup-ops/fail-tap.ts new file mode 100644 index 0000000..f9f40dc --- /dev/null +++ b/conformance/runners/typescript/cup-ops/fail-tap.ts @@ -0,0 +1,14 @@ +import type { Payload, Tap } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** throw */ +export class FailTap implements Tap { + readonly message: string; + + constructor(message: string) { + this.message = message; + } + + observe(_payload: Payload): void { + throw new Error(this.message); + } +} diff --git a/conformance/runners/typescript/cup-ops/has-filter.ts b/conformance/runners/typescript/cup-ops/has-filter.ts new file mode 100644 index 0000000..6527f5f --- /dev/null +++ b/conformance/runners/typescript/cup-ops/has-filter.ts @@ -0,0 +1,16 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert to = has(key) */ +export class HasFilter implements Filter { + readonly key: string; + readonly to: string; + + constructor(key: string, to: string) { + this.key = key; + this.to = to; + } + + call(payload: Payload): Payload { + return payload.insert(this.to, payload.has(this.key)); + } +} diff --git a/conformance/runners/typescript/cup-ops/merge-filter.ts b/conformance/runners/typescript/cup-ops/merge-filter.ts new file mode 100644 index 0000000..8841cc6 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/merge-filter.ts @@ -0,0 +1,15 @@ +import { Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; +import type { Filter } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** merge(Payload(value)) */ +export class MergeFilter implements Filter { + readonly value: Record; + + constructor(value: Record) { + this.value = value; + } + + call(payload: Payload): Payload { + return payload.merge(new Payload(this.value)); + } +} diff --git a/conformance/runners/typescript/cup-ops/mutate-filter.ts b/conformance/runners/typescript/cup-ops/mutate-filter.ts new file mode 100644 index 0000000..9acc3a2 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/mutate-filter.ts @@ -0,0 +1,17 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert(key, value) as a statement of its own, then return the payload */ +export class MutateFilter implements Filter { + readonly key: string; + readonly value: unknown; + + constructor(key: string, value: unknown) { + this.key = key; + this.value = value; + } + + call(payload: Payload): Payload { + payload.insert(this.key, this.value); + return payload; + } +} diff --git a/conformance/runners/typescript/cup-ops/nested-pipeline.ts b/conformance/runners/typescript/cup-ops/nested-pipeline.ts new file mode 100644 index 0000000..6db705d --- /dev/null +++ b/conformance/runners/typescript/cup-ops/nested-pipeline.ts @@ -0,0 +1,4 @@ +import { Pipeline } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** A named pipeline type, for a pipeline that is a step of another. */ +export class NestedPipeline extends Pipeline {} diff --git a/conformance/runners/typescript/cup-ops/noop-tap.ts b/conformance/runners/typescript/cup-ops/noop-tap.ts new file mode 100644 index 0000000..b9fbb55 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/noop-tap.ts @@ -0,0 +1,6 @@ +import type { Payload, Tap } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** do nothing */ +export class NoopTap implements Tap { + observe(_payload: Payload): void {} +} diff --git a/conformance/runners/typescript/cup-ops/record-hook.ts b/conformance/runners/typescript/cup-ops/record-hook.ts new file mode 100644 index 0000000..057fced --- /dev/null +++ b/conformance/runners/typescript/cup-ops/record-hook.ts @@ -0,0 +1,26 @@ +import { Hook } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** + * Records before:, after: and error:. Events of the + * pipeline itself have an empty step. + */ +export class RecordHook extends Hook { + readonly events: string[]; + + constructor(events: string[]) { + super(); + this.events = events; + } + + override before(step: string | null): void { + this.events.push("before:" + (step ?? "")); + } + + override after(step: string | null): void { + this.events.push("after:" + (step ?? "")); + } + + override onError(step: string | null): void { + this.events.push("error:" + (step ?? "")); + } +} diff --git a/conformance/runners/typescript/cup-ops/record-tap.ts b/conformance/runners/typescript/cup-ops/record-tap.ts new file mode 100644 index 0000000..b6105f4 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/record-tap.ts @@ -0,0 +1,15 @@ +import type { Payload, Tap } from "../../../../packages/codeupipe-ts/src/index.ts"; +import { TAPS } from "./sink.ts"; + +/** append get(key) to the runner's tap log */ +export class RecordTap implements Tap { + readonly key: string; + + constructor(key: string) { + this.key = key; + } + + observe(payload: Payload): void { + TAPS.push(payload.get(this.key)); + } +} diff --git a/conformance/runners/typescript/cup-ops/remove-filter.ts b/conformance/runners/typescript/cup-ops/remove-filter.ts new file mode 100644 index 0000000..462c657 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/remove-filter.ts @@ -0,0 +1,14 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** remove(key) */ +export class RemoveFilter implements Filter { + readonly key: string; + + constructor(key: string) { + this.key = key; + } + + call(payload: Payload): Payload { + return payload.remove(this.key); + } +} diff --git a/conformance/runners/typescript/cup-ops/run-again-filter.ts b/conformance/runners/typescript/cup-ops/run-again-filter.ts new file mode 100644 index 0000000..08f0ea2 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/run-again-filter.ts @@ -0,0 +1,25 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +import { RUN_AGAIN } from "./sink.ts"; + +/** + * The reference `run_again` step (conformance/README.md): it starts a second + * run of the pipeline that it is in, inside the first one. How a pipeline is + * run again depends on the target, so the runner gives that as a function, + * in `RUN_AGAIN`. The second run has `inner` in its input, so it does not + * start a third one. + */ +export class RunAgainFilter implements Filter { + readonly data: Record; + + constructor(data: Record) { + this.data = data; + } + + async call(payload: Payload): Promise { + if (payload.get("inner") !== true) { + await RUN_AGAIN.run({ ...this.data }); + } + return payload; + } +} diff --git a/conformance/runners/typescript/cup-ops/set-filter.ts b/conformance/runners/typescript/cup-ops/set-filter.ts new file mode 100644 index 0000000..75fe72a --- /dev/null +++ b/conformance/runners/typescript/cup-ops/set-filter.ts @@ -0,0 +1,16 @@ +import type { Filter, Payload } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** insert key = value */ +export class SetFilter implements Filter { + readonly key: string; + readonly value: unknown; + + constructor(key: string, value: unknown) { + this.key = key; + this.value = value; + } + + call(payload: Payload): Payload { + return payload.insert(this.key, this.value); + } +} diff --git a/conformance/runners/typescript/cup-ops/sink.ts b/conformance/runners/typescript/cup-ops/sink.ts new file mode 100644 index 0000000..f017898 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/sink.ts @@ -0,0 +1,10 @@ +/** Where the Taps of the runner record what they see. The runner empties it before each case. */ +export const TAPS: unknown[] = []; + +/** The State of each run that a `run_again` step started, in order. */ +export const INNER_STATES: Array<{ executed: string[]; skipped: string[]; errors: Array<[string, unknown]> }> = []; + +/** How the target runs its pipeline one more time. The runner sets `run` for each case. */ +export const RUN_AGAIN: { run: (input: Record) => Promise } = { + run: async () => { throw new Error("the runner did not say how to run the pipeline again"); }, +}; diff --git a/conformance/runners/typescript/cup-ops/truthy-predicate.ts b/conformance/runners/typescript/cup-ops/truthy-predicate.ts new file mode 100644 index 0000000..066cb51 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/truthy-predicate.ts @@ -0,0 +1,14 @@ +import type { Payload, Predicate } from "../../../../packages/codeupipe-ts/src/index.ts"; + +/** get(key) is truthy; a missing key is false */ +export class TruthyPredicate implements Predicate { + readonly key: string; + + constructor(key: string) { + this.key = key; + } + + test(payload: Payload): boolean { + return Boolean(payload.get(this.key)); + } +} diff --git a/conformance/runners/typescript/package.json b/conformance/runners/typescript/package.json new file mode 100644 index 0000000..e986b24 --- /dev/null +++ b/conformance/runners/typescript/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/conformance/runners/typescript/runner.ts b/conformance/runners/typescript/runner.ts new file mode 100644 index 0000000..d833f9e --- /dev/null +++ b/conformance/runners/typescript/runner.ts @@ -0,0 +1,255 @@ +/** + * TypeScript conformance runner: runs the cases and reports what happened. + * + * node runner.ts < cases.json > results.json + * + * Targets + * run the Pipeline of packages/codeupipe-ts + * static the same pipeline as plain code from packages/codeupipe-compile-ts, + * written to a file and imported from it + * + * The runner never judges; conformance/run.py compares the results of every + * target with the same expectations. Node runs this file as it is (Node + * 22.18 or later removes the types). The static target needs the compile + * package to be installed (npm ci in packages/codeupipe-compile-ts). Plain + * code has no State and no Hooks, so the static target reports neither. + */ +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { Payload, Pipeline, State } from "../../../packages/codeupipe-ts/src/index.ts"; +import type { Filter, Predicate, Tap } from "../../../packages/codeupipe-ts/src/index.ts"; +import { AddFilter } from "./cup-ops/add-filter.ts"; +import { AppendFilter } from "./cup-ops/append-filter.ts"; +import { CopyFilter } from "./cup-ops/copy-filter.ts"; +import { EqualsPredicate } from "./cup-ops/equals-predicate.ts"; +import { FailFilter } from "./cup-ops/fail-filter.ts"; +import { FailTap } from "./cup-ops/fail-tap.ts"; +import { HasFilter } from "./cup-ops/has-filter.ts"; +import { MergeFilter } from "./cup-ops/merge-filter.ts"; +import { MutateFilter } from "./cup-ops/mutate-filter.ts"; +import { NestedPipeline } from "./cup-ops/nested-pipeline.ts"; +import { NoopTap } from "./cup-ops/noop-tap.ts"; +import { RecordHook } from "./cup-ops/record-hook.ts"; +import { RecordTap } from "./cup-ops/record-tap.ts"; +import { RemoveFilter } from "./cup-ops/remove-filter.ts"; +import { SetFilter } from "./cup-ops/set-filter.ts"; +import { RunAgainFilter } from "./cup-ops/run-again-filter.ts"; +import { INNER_STATES, RUN_AGAIN, TAPS } from "./cup-ops/sink.ts"; +import { TruthyPredicate } from "./cup-ops/truthy-predicate.ts"; + +type Spec = Record; +type Case = { id: string; hooks?: boolean; given_state?: boolean; steps: Spec[]; input: Record }; + +class Unsupported extends Error {} + +function makeFilter(spec: Spec): Filter { + switch (spec.op) { + case "set": return new SetFilter(spec.key, spec.value); + case "copy": return new CopyFilter(spec.from, spec.to); + case "add": return new AddFilter(spec.key, spec.by); + case "append": return new AppendFilter(spec.key, spec.value); + case "remove": return new RemoveFilter(spec.key); + case "merge": return new MergeFilter(spec.value); + case "has": return new HasFilter(spec.key, spec.to); + case "mutate": return new MutateFilter(spec.key, spec.value); + case "fail": return new FailFilter(spec.message); + case "run_again": return new RunAgainFilter(spec.input); + default: throw new Unsupported(`unknown filter op '${spec.op}'`); + } +} + +function makeTap(spec: Spec): Tap { + switch (spec.op) { + case "record": return new RecordTap(spec.key); + case "fail": return new FailTap(spec.message); + case "noop": return new NoopTap(); + default: throw new Unsupported(`unknown tap op '${spec.op}'`); + } +} + +function makePredicate(when: Spec): Predicate { + switch (when.op) { + case "equals": return new EqualsPredicate(when.key, when.value); + case "truthy": return new TruthyPredicate(when.key); + default: throw new Unsupported(`unknown valve op '${when.op}'`); + } +} + +function build(steps: Spec[], pipeline: Pipeline): Pipeline { + for (const step of steps) { + if (step.kind === "filter") { + pipeline.addFilter(makeFilter(step)); + } else if (step.kind === "tap") { + pipeline.addTap(makeTap(step)); + } else if (step.kind === "valve") { + pipeline.addValve(makePredicate(step.when), makeFilter(step.filter)); + } else if (step.kind === "pipeline") { + pipeline.addPipeline(build(step.steps, new NestedPipeline())); + } else { + throw new Unsupported(`unknown step kind '${step.kind}'`); + } + } + return pipeline; +} + +/** + * Label to the step name of the case. The cases name their steps, and the + * runtime reports labels (conformance/API.md). The steps are added in the + * order of the case, so the two lists are side by side. A label that the + * runtime reports and that is not in this map stays as it is, and fails the + * comparison. + */ +function namesByLabel(pipeline: Pipeline, steps: Spec[]): Map { + return new Map(pipeline.labels.map((label, index) => [label, String(steps[index]?.name)])); +} + +async function runCase(spec: Case): Promise> { + const events: string[] = []; + const pipeline = build(spec.steps, new Pipeline()); + const names = namesByLabel(pipeline, spec.steps); + const named = (label: string) => names.get(label) ?? label; + if (spec.hooks) pipeline.useHook(new RecordHook(events)); + + const data = structuredClone(spec.input); + const result: Record = { id: spec.id }; + // With "given_state", the caller gives the State of the run. If not, the run makes one. + const given = spec.given_state ? new State() : undefined; + RUN_AGAIN.run = (input) => { + const state = new State(); + INNER_STATES.push(state); + return pipeline.run(new Payload(input), state); + }; + try { + result.output = (await (given === undefined ? pipeline.run(new Payload(data)) : pipeline.run(new Payload(data), given))).toDict(); + } catch (error) { + result.error = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + } + const report = (state: { executed: string[]; skipped: string[]; errors: Array<[string, unknown]> }) => ({ + executed: state.executed.map(named), + skipped: state.skipped.map(named), + errors: state.errors.map(([label]) => named(label)), + }); + result.state = report(given ?? pipeline.state); + result.pipeline_state = report(pipeline.state); + result.inner_states = INNER_STATES.map(report); + result.taps = [...TAPS]; + if (spec.hooks) { + result.hooks = events.map((event) => { + const at = event.indexOf(":"); + return event.slice(0, at + 1) + named(event.slice(at + 1)); + }); + } + result.input_after = data; + return result; +} + +// -- the static target -------------------------------------------------- + +const HERE = import.meta.dirname; +const CLASS_OF: Record = { + set: ["SetFilter", "set-filter"], copy: ["CopyFilter", "copy-filter"], add: ["AddFilter", "add-filter"], + append: ["AppendFilter", "append-filter"], remove: ["RemoveFilter", "remove-filter"], + merge: ["MergeFilter", "merge-filter"], has: ["HasFilter", "has-filter"], mutate: ["MutateFilter", "mutate-filter"], + fail: ["FailFilter", "fail-filter"], run_again: ["RunAgainFilter", "run-again-filter"], record: ["RecordTap", "record-tap"], "tap:fail": ["FailTap", "fail-tap"], + noop: ["NoopTap", "noop-tap"], equals: ["EqualsPredicate", "equals-predicate"], truthy: ["TruthyPredicate", "truthy-predicate"], +}; +const ARGUMENTS: Record = { + set: ["key", "value"], copy: ["from", "to"], add: ["key", "by"], append: ["key", "value"], remove: ["key"], + merge: ["value"], has: ["key", "to"], mutate: ["key", "value"], fail: ["message"], run_again: ["input"], record: ["key"], + "tap:fail": ["message"], noop: [], equals: ["key", "value"], truthy: ["key"], +}; + +/** The case as the source of a pipeline class: the form that the compiler reads. */ +function sourceOf(spec: Case): string { + const imports = new Map(); + const classes: string[] = []; + const made = (op: string, step: Spec): string => { + const entry = CLASS_OF[op]; + if (entry === undefined) throw new Unsupported(`unknown op '${op}'`); + imports.set(entry[0], join(HERE, "cup-ops", entry[1] + ".ts")); + return `new ${entry[0]}(${(ARGUMENTS[op] ?? []).map((name) => JSON.stringify(step[name])).join(", ")})`; + }; + const pipelineClass = (name: string, steps: Spec[]): void => { + const lines: string[] = []; + for (const step of steps) { + if (step.kind === "filter") { + lines.push(`this.addFilter(${made(step.op, step)});`); + } else if (step.kind === "tap") { + lines.push(`this.addTap(${made(step.op === "fail" ? "tap:fail" : step.op, step)});`); + } else if (step.kind === "valve") { + lines.push(`this.addValve(${made(step.when.op, step.when)}, ${made(step.filter.op, step.filter)});`); + } else if (step.kind === "pipeline") { + const inner = `Inner${classes.length + 1}Pipeline`; + classes.push(""); // keeps the place, so that each nested class has its own number + pipelineClass(inner, step.steps); + lines.push(`this.addPipeline(new ${inner}());`); + } else { + throw new Unsupported(`unknown step kind '${step.kind}'`); + } + } + classes.push(`export class ${name} extends Pipeline {\n constructor() {\n super();\n${lines.map((line) => " " + line).join("\n")}\n }\n}`); + }; + pipelineClass("CasePipeline", spec.steps); + const head = [`import { Pipeline } from ${JSON.stringify(join(HERE, "../../../packages/codeupipe-ts/src/index.ts"))};`, + ...[...imports].map(([name, file]) => `import { ${name} } from ${JSON.stringify(file)};`)]; + return [...head, "", ...classes.filter((text) => text !== "")].join("\n") + "\n"; +} + +type Compile = (file: string, className: string, options: { outFile: string }) => Promise; + +async function runStatic(spec: Case, compile: Compile, directory: string, index: number): Promise> { + const sourceFile = join(directory, `case-${index}-pipeline.ts`); + const outFile = join(directory, `case-${index}-compiled.ts`); + writeFileSync(sourceFile, sourceOf(spec)); + try { + await compile(sourceFile, "CasePipeline", { outFile }); + } catch (error) { + if (error instanceof Error && error.name === "CompileRefusal") throw new Unsupported(error.message); + throw error; + } + const compiled: { run(input: Record): Promise> } = await import(pathToFileURL(outFile).href); + + // The compiled form is a function. To run the pipeline again is to call that function again. + RUN_AGAIN.run = (input) => compiled.run(input); + const data = structuredClone(spec.input); + const result: Record = { id: spec.id }; + try { + result.output = await compiled.run(data); + } catch (error) { + result.error = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + } + result.taps = [...TAPS]; + result.input_after = data; + return result; +} + +// -- main ----------------------------------------------------------------- + +const target = process.argv[2]; +if (target !== "run" && target !== "static") { + console.error(`unknown target '${target}'. The targets are: run, static`); + process.exit(2); +} +let compile: Compile | undefined; +let directory = ""; +if (target === "static") { + ({ compile } = await import("../../../packages/codeupipe-compile-ts/src/index.ts")); + directory = mkdtempSync(join(tmpdir(), "cup-conformance-")); +} + +const cases: Case[] = JSON.parse(readFileSync(0, "utf8")); +const results: Record[] = []; +for (const [index, spec] of cases.entries()) { + TAPS.length = 0; + INNER_STATES.length = 0; + try { + results.push(compile === undefined ? await runCase(spec) : await runStatic(spec, compile, directory, index)); + } catch (error) { + if (!(error instanceof Unsupported)) throw error; + results.push({ id: spec.id, unsupported: error.message }); + } +} +process.stdout.write(JSON.stringify(results)); diff --git a/cup.config.json b/cup.config.json index e2b5f45..8da157a 100644 --- a/cup.config.json +++ b/cup.config.json @@ -1,6 +1,6 @@ { "lint": { - "paths": ["packages/codeupipe/codeupipe", "packages/codeupipe-lint/codeupipe_lint", "scorecard", "conformance/runners/python", "examples"], + "paths": ["packages/codeupipe/codeupipe", "packages/codeupipe-lint/codeupipe_lint", "scorecard", "conformance/runners/python", "conformance/judging", "conformance/run.py", "examples"], "exclude": ["*/tests/lint/fixtures/*"] } } diff --git a/docs/static-export.md b/docs/compile.md similarity index 67% rename from docs/static-export.md rename to docs/compile.md index b032f8d..a0fc93d 100644 --- a/docs/static-export.md +++ b/docs/compile.md @@ -1,18 +1,18 @@ -# Static export: what it guarantees +# Compile: what it guarantees -`codeupipe.lower` turns a live `Pipeline` into plain Python with no framework -import. This page says what the export lowers, what it refuses, and what -the exported code does not have. +`codeupipe.compile` turns a live `Pipeline` into plain Python with no framework +import. This page says what the compiler lowers, what it refuses, and what +the compiled code does not have. -**The rule: anything that exports behaves the same as the dynamic `run`. -Anything that cannot is refused at export time, with the step named.** A -shape that exports "successfully" and then behaves differently is a bug. +**The rule: anything that compiles behaves the same as the dynamic `run`. +Anything that cannot is refused at compile time, with the step named.** A +shape that compiles "successfully" and then behaves differently is a bug. -There are two exports. Pick per pipeline: +There are two modes. Pick per pipeline: | | fast (default) | observable (`observe=True`) | |---|---|---| -| exported entry point | `run(data)` | `run(data, state=None, hooks=None, timing=False, trace_id=None)` | +| compiled entry point | `run(data)` | `run(data, state=None, hooks=None, timing=False, trace_id=None)` | | output | same as dynamic | same as dynamic | | State (executed, skipped, errors, timings) | no | yes, into the `state` dict | | Hooks | refused | yes, same order as dynamic | @@ -22,7 +22,7 @@ There are two exports. Pick per pipeline: | 10 filters + a Hook (dynamic: 5.96) | refused | 3.03 | ```python -from codeupipe.lower import build_export_pipeline_v2 +from codeupipe.compile import build_export_pipeline_v2 out = build_export_pipeline_v2(write=False).run(Payload({ "pipeline": pipeline, @@ -34,9 +34,9 @@ out.get("files") # [{"path": ..., "content": ...}], exposing run(di ## From the command line ```bash -python -m codeupipe.lower app.pipelines:checkout --out build/checkout -python -m codeupipe.lower app.pipelines:checkout --out build/checkout --lowering inline -python -m codeupipe.lower app.pipelines:checkout --out build/checkout --observe +python -m codeupipe.compile app.pipelines:checkout --out build/checkout +python -m codeupipe.compile app.pipelines:checkout --out build/checkout --lowering inline +python -m codeupipe.compile app.pipelines:checkout --out build/checkout --observe ``` `module:name` is a `Pipeline`, a function that builds one, or a named @@ -46,19 +46,19 @@ file written is printed. A directory that is not empty is refused unless steps deleted from the pipeline leave their old modules behind). With no `module:name`, every pipeline listed under `compile` in -`cup.config.json` is exported ([config.md](config.md)). +`cup.config.json` is compiled ([config.md](config.md)). -Exit code: **0** exported, **1** refused (the message names the step and the +Exit code: **0** compiled, **1** refused (the message names the step and the fix), **2** the command could not run. Nothing is written on 1 or 2. Tests: -[tests/lower/test_cli.py](../tests/lower/test_cli.py). +[tests/compile/test_cli.py](../tests/compile/test_cli.py). -## Step identity in exported code +## Step labels in compiled code -Steps are identified by their filter (`app.filters.Validate`, `#2` from the -second use; conformance/API.md). An identity is a label, not an identifier, -so exported code never uses it as a name: each step gets a generated module -and function, `step_01_validate`, `step_02_validate`, and the identity -appears only as a string in State and in Hook calls. That also rules out +A step has a label, the name of its class (`Validate`, `Validate#2` from the +second use; conformance/API.md). A label is not an identifier, so compiled +code never uses it as a name: each step gets a generated module and +function, `step_01_validate`, `step_02_validate`, and the label appears only +as a string in State and in Hook calls. That also rules out every collision between a step and a builtin, a module global or the generated `run()` (scorecard findings F3–F8). @@ -71,31 +71,32 @@ generated `run()` (scorecard findings F3–F8). | Tap (`observe`) | the call wrapped in `try/except Exception` (Plane B: never halts) | conformance, `test_export_pipeline` | | Nested Pipeline (a named `Pipeline` type) | its steps flattened in place; reported as the one nesting step | conformance, `test_export_pipeline` | | Literal instance settings (`self.key` holding a str, number, ...) | compiled in as constants | `test_export_instance_config` | -| Type hints on a step (`Payload[Reads] -> Payload[Writes]`, hinted locals) | erased: exported code carries no hints and never imports their names | `test_export_typed_steps` | +| Type hints on a step (`Payload[Reads] -> Payload[Writes]`, hinted locals) | erased: compiled code carries no hints and never imports their names | `test_export_typed_steps` | | Payload ops `get insert remove merge has to_dict copy` | dict operations; a write is an in-place statement (`data[k] = v`) | `test_rewrite_payload_ops` | +| The Payload parameter under any name (`call(self, order)`, `test(self, p)`) | the same, followed by its name: an unrelated local called `payload` stays separate | `test_export_parameter_name` | -Evidence: all 27 conformance cases on both layouts (`python:static-steps`, +Evidence: all 30 conformance cases on both layouts (`python:static-steps`, `python:static-inline`), and the parity spike at 2400/2400 inputs. ## What is refused -`UnsupportedFilterShape` is raised at export, naming the step. Tests: -[tests/lower/test_export_refusals.py](../tests/lower/test_export_refusals.py) +`UnsupportedFilterShape` is raised at compile time, naming the step. Tests: +[tests/compile/test_export_refusals.py](../tests/compile/test_export_refusals.py) and the older shape tests beside it. | Shape | Why there is no static form | What to do | |---|---|---| -| **Hooks**, in the fast export | the fast code has no lifecycle to hook into | export with `observe=True`, or remove them | -| **Hooks on a nested pipeline**, in either export | they belong to the inner run, which is flattened away | move them to the top-level pipeline | +| **Hooks**, in the fast mode | the fast code has no lifecycle to hook into | compile with `observe=True`, or remove them | +| **Hooks on a nested pipeline**, in either mode | they belong to the inner run, which is flattened away | move them to the top-level pipeline | | `observe=True` with `lowering="inline"` (`ValueError`) | inline writes the payload in place, so a Hook holding it would see later steps' writes | use the default `steps` layout | | **Async steps** (async `call`/`observe`, Valve around one, async `test`) | static `run` is sync, as the dynamic `run` is: it raises `AsyncStepInSyncRun` on the same pipeline | make the step sync | | **`payload.trace_id`, `payload.lineage`, `payload.with_trace`** | a plain dict carries neither | keep the step dynamic | -| any other payload attribute (`payload._data`, ...) | same: only the operations above have a dict form | use an exportable operation | +| any other attribute of the Payload parameter (`payload._data`, `order.price` on a type of your own, ...) | same: only the operations above have a dict form. The refusal names the parameter as the step wrote it | use a compilable operation, or keep a step over a type of your own dynamic | | callables or objects on `self` (filter or predicate), steps writing `self.x` | no source or literal to compile in | literal settings (predicates and filters are module-level classes anyway: the runtime refuses anything else) | -## What the fast export gives up +## What the fast mode gives up -The fast export produces the same **output**. It does not produce: +The fast mode produces the same **output**. It does not produce: - **State**: no executed / skipped / errors / timings. - **Hook calls** (refused, above). @@ -105,7 +106,7 @@ The fast export produces the same **output**. It does not produce: So conformance can judge the fast targets only on output, Taps and an unchanged caller input: they pass the State and Hook cases as ✅\* (partial). -## Observable export (`observe=True`) +## Observable mode (`observe=True`) ```python out = build_export_pipeline_v2(write=False).run(Payload({"pipeline": pipeline, "observe": True})) @@ -116,7 +117,7 @@ state["executed"], state["skipped"], state["errors"], state["timings"], state["t ``` It reproduces the dynamic run's observable behaviour exactly, and the -`python:static-observe` conformance target checks it on all 27 cases: +`python:static-observe` conformance target checks it on all 30 cases: - **State** is written into the `state` dict you pass (reset each run): `executed`, `skipped`, `errors` as `(name, error)`, `timings` (only with @@ -127,7 +128,7 @@ It reproduces the dynamic run's observable behaviour exactly, and the contract's order (conformance/API.md). They receive the **dict**, not a `Payload`: `.get` works, but treat it as read-only, and `trace_id` is on `state`, not on the data. -- **Hooks are passed at run time**, since the exported code imports no +- **Hooks are passed at run time**, since the compiled code imports no framework and cannot rebuild hook objects. If the source pipeline had Hooks, the module lists them in `SOURCE_HOOKS` and `run()` **raises `TypeError` if `hooks` is omitted**, so they cannot be lost silently. @@ -143,15 +144,15 @@ Cost: 10 filters run in 1.95 µs against 0.84 for the fast per-step layout and 4.92 for the dynamic run: about 2.5× faster than dynamic with the same State and Hooks. With a Hook attached, 3.03 µs against 5.96. -## Testing exported code +## Testing compiled code -- **Fast export:** the package is `run(dict) -> dict`, so test by output. +- **Fast mode:** the package is `run(dict) -> dict`, so test by output. The scorecard (`python -m scorecard.differential`) runs the dynamic and static forms on the same inputs and compares outputs, raised errors and input mutation. -- **Observable export:** assert on the `state` dict and on a recording Hook, +- **Observable mode:** assert on the `state` dict and on a recording Hook, exactly as you would on `pipeline.state` and a Hook in the dynamic - runtime. `tests/lower/test_export_observe.py` runs both forms side by side + runtime. `tests/compile/test_export_observe.py` runs both forms side by side and asserts they match. - `run_filter` and `assert_state` from `codeupipe.testing` target the dynamic runtime only. diff --git a/docs/config.md b/docs/config.md index a5ad3bb..286a0f7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -22,7 +22,7 @@ every time. ```bash python -m codeupipe_lint.invariants # lint.paths, minus lint.exclude python -m codeupipe_lint.anonymous_steps # the same -python -m codeupipe.lower # every pipeline under compile +python -m codeupipe.compile # every pipeline under compile ``` ## Keys diff --git a/docs/rust-plan.md b/docs/rust-plan.md new file mode 100644 index 0000000..df83672 --- /dev/null +++ b/docs/rust-plan.md @@ -0,0 +1,544 @@ +# Plan: codeupipe for Rust + +This is a plan, not a reference. It says how the third runtime is added: a +Rust runtime with the same contract ([conformance/API.md](../conformance/API.md)), +judged by the same conformance cases, in the same layout as TypeScript. + +**How it is built: from the contract, tests first.** Each behaviour starts as +a failing test that comes from a rule of the contract or from a test of the +TypeScript package. **When in doubt, do what the TypeScript package does**: it +is the reference for every question that the contract does not answer. + +The Rust runtime in the old repository (`runtimes/rs`) is not the start. Its +API is from before version 2: steps named by hand, no Predicate, no labels. +It can be read for ideas. No code is copied from it. + +## Versions + +The Rust packages start at **0.1.0**, as the TypeScript packages did. A version +below 1.0 is not stable yet, and a new version can change in a way that is not +compatible. For Cargo that means: 0.1.x to 0.2.0 is a change that is not +compatible. + +The packages are not on crates.io at first. Each has `publish = false`, as each +TypeScript package has `"private": true`. Until they are published, a project +uses them from this repository: + +```toml +[dependencies] +codeupipe = { git = "https://github.com/orchestrate-solutions/codeupipe", branch = "main" } +``` + +Cargo finds the crate by its name inside the repository, so this works with +the crate in `packages/codeupipe-rs`. + +## Layout + +The packages are beside the others, with `-rs` where TypeScript has `-ts`. + +| Path | Crate | Contents | Phase | +|---|---|---|---| +| `packages/codeupipe-rs/` | `codeupipe` | The eight primitives. No dependencies | 1 | +| `conformance/runners/rust/` | (binary, not published) | The conformance runner, with its `cup_ops` steps | 2 | +| `packages/codeupipe-compile-rs/` | `codeupipe-compile` | Writes a pipeline as plain Rust with no use of codeupipe | 3 | +| `packages/codeupipe-lint-rs/` | `codeupipe-lint` | The linter for rules that the type system does not already check | 4 | + +A root `Cargo.toml` makes the Rust crates one Cargo workspace. It is not a +package, as the root `package.json` is not a package. `Cargo.lock` is +committed, since the workspace has a binary (the runner). `target/` goes in +`.gitignore`. The linter (phase 4) is a workspace of its own, with its own +`Cargo.lock`: it needs `syn` from crates.io, and the root workspace builds +with no download. + +One primitive is in each file, and the file name is the type name in +snake_case, as in Python: `DiscountFilter` is in `discount_filter.rs`. + +### The core crate + +``` +packages/codeupipe-rs/ + Cargo.toml name = "codeupipe", version = "0.1.0", publish = false + README.md LICENSE NOTICE + src/ + lib.rs the public names, as index.ts has them + value.rs Value: null, bool, number, string, list, map + payload.rs Payload + filter.rs trait Filter + predicate.rs trait Predicate + tap.rs trait Tap + hook.rs trait Hook, every method with a default that does nothing + state.rs State + valve.rs Valve (not a public name) + pipeline.rs Pipeline + named_pipeline.rs trait NamedPipeline (see decision D1) + step_label.rs the label from the type name + step_error.rs the error type of a step (see decision D4) + tests/ + payload.rs pipeline.rs step_label.rs pricing_example.rs + examples/pricing/ + main.rs discount_filter.rs mark_bulk_filter.rs at_least_predicate.rs + audit_tap.rs print_steps_hook.rs pricing_pipeline.rs +``` + +## Decisions + +These are the questions that Rust raises and Python and TypeScript do not. +**They are settled as recommended** (2026-10-27). + +### D1. A named pipeline + +**The question.** The contract says to make a nested pipeline a named type +(`class UserOnboarding(Pipeline)`), so that its label says what it is. Rust has +no classes to extend. + +**Recommendation.** A trait that a named type implements. Its one method adds +the steps. The label is the name of the type. + +```rust +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} + +let mut pipeline = Pipeline::named(PricingPipeline); +``` + +`add_steps` is also what the compiler reads (phase 3), as the TypeScript +compiler reads the constructor. A plain `Pipeline::new()` has the label +`Pipeline`, as in the contract. + +### D2. How a write returns the Payload + +**The question.** The contract says that a write changes the Payload and +returns it, so that `return payload.insert(k, v)` is one line. + +**Settled.** A Filter gets `&mut Payload` and writes in place. A write takes +`&mut self` and returns it, so writes chain: +`payload.insert("a", 1).remove("b")`. + +```rust +fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0); + payload.insert("price", (price * (1.0 - self.rate) * 100.0).round() / 100.0); + Ok(()) +} +``` + +The first recommendation was by value: `call(payload: Payload) -> Result`. +It was changed in phase 1, against the TypeScript package. TypeScript changes +the Payload in place, and its `onError` gets that same Payload, with the +writes the failing step made before it threw. By value, the Payload is gone +when the Filter fails, and `on_error` has none to give. `&mut` does what +TypeScript does, and a Filter cannot return a different Payload (which the +TypeScript compiler has to refuse). + +A Tap, a Predicate and a Hook get `&Payload`. So **only a Filter can write, and +the compiler checks it**: lint rule `CUP-I004` (a write outside a Filter) is a +type error in Rust, not a lint finding. + +### D3. `get` with a default + +**The question.** The contract has `get(k, default)`. Rust has no default +arguments and no overloads. + +**Recommendation.** `get(k) -> Option<&Value>`. A default is +`.unwrap_or(...)`, which is the language's own way, not a second name. +`Value` has `as_f64`, `as_str`, `as_bool`, `as_list`, `as_map`. Absent is +`None`. + +### D4. Errors + +**The question.** A Filter failure raises, the Hook gets "the error itself", +and State keeps (label, error). + +**Recommendation.** A step returns `Result`. `StepError` is +`Box`. `on_error` gets `&StepError`, and +then `run` returns the same error. A Tap error goes into `state.errors`. + +A **panic is not a failure**. It is not caught: a panicking Tap stops the run. +The README says so. Catching panics (`catch_unwind`) would make the Tap rule +hold for panics too, but it needs `UnwindSafe` bounds on every step. It can +be added later without changing the API. + +### D5. Labels + +**The question.** The label is the name of the step's type. + +**Recommendation.** `add_filter(&mut self, filter: F)` +reads `std::any::type_name::()` when the step is added, and keeps the last +part of the path (`examples::pricing::DiscountFilter` gives `DiscountFilter`). +Generic arguments are dropped (`Wrapper` gives `Wrapper`). A closure +does not implement `Filter`, so it cannot be a step. There is no blanket +implementation for `Fn`. + +What the contract refuses with a `TypeError` when a step is added is a +compile error in Rust: a value without `call` does not implement `Filter`, and +`add_valve(filter, predicate)` does not type-check. Tests for these use +`compile_fail` doc tests, which need no dependency. + +### D6. Values and the order of keys + +**The question.** Values are JSON-shaped, and the core crate has no +dependencies, so there is no `serde_json::Value`. + +**Recommendation.** An own `Value` enum: `Null`, `Bool`, `Number(f64)`, +`String`, `List(Vec)`, `Map(...)`. One number type, as JavaScript has, +and as the conformance judge compares (`1 == 1.0`). + +Keys keep the order in which they were inserted, as a Python dict and a +JavaScript Map do. `to_dict` gives them in that order. The judge does not +compare order, so this is for people, not for conformance. + +`copy()` is a deep copy in this version. The contract says the two Payloads +"share their values", but in Rust a shared value cannot be changed without a +`Cell` or a lock, so a deep copy shows the same behaviour. Sharing with `Rc` +can be added later for speed, without a change to the API. + +### D7. Sync and async + +**Recommendation.** 0.1.0 has `run` only, and it is sync (contract principle +3). A sync caller never pays for an executor. `run_async` waits for a +conformance case that needs it, and for a choice of how to stay free of +dependencies (an `async fn` in the traits, driven by the caller's executor). + +### D8. State and the run + +**Changed (2026-10-27): `run(&self, ...)`**, see "A State for each run" +below. The first recommendation was: + +`run(&mut self, input: &Payload) -> Result`. +It copies `input`, so the caller's Payload is not changed (contract), and it +writes a new `State` into the pipeline. `pipeline.state()` reads it. +`&mut self` means that two runs of one pipeline at the same time are not +possible. That is simpler than the TypeScript rule ("the run that started +last") and it needs no lock. + +`timings` keeps a `Duration` for each label. `observe(timing, lineage)` takes +two `bool`s. + +On `wasm32-unknown-unknown`, `std::time::Instant` panics. Timing is off by +default, so a pipeline runs in WebAssembly. Timing there is out of scope for +0.1.0, and the README says so. + +### D9. Edition and Rust version + +**Recommendation.** Edition 2021, `rust-version = "1.75"`. The core uses +nothing newer. This machine has 1.89. + +### D10. The crate names + +`codeupipe`, `codeupipe-compile` and `codeupipe-lint` are the names in the +other languages. Before the first publish, check that they are free on +crates.io. The old repository used `codeupipe-core`. + +### D11. A payload of your own + +**Settled (2026-10-27).** `Payload` stays the default: a map that any +developer can read, which shows that anything can go there. A developer, or +an agent, that would write a struct anyway can opt in: each primitive takes +the data type as a parameter with `Payload` as its default (`Filter

`, `Pipeline

`), and the type implements `PipelineData`: +`Clone` (for the copy that `run` works on) and an optional `stamp` for +lineage. Code written for `Payload` does not change; `Pipeline::new()` infers +the type from its steps. + +Python and TypeScript have the same option (2026-10-27): Python takes any +object with `copy()`, and TypeScript has the same type parameter. In all +three, lineage is kept only by a type that can keep it, and does nothing for +any other ("A type of your own" in conformance/API.md). A step for one +type in a pipeline of another is a compile error, checked by a doc test. + +Measured (release, Apple Silicon): the pricing pipeline runs in 0.024 µs +with an `Order` struct against 0.18 µs with a Payload, and 50 writes take +0.46 µs against 4.3 µs. What is left, about 9 ns a step, is the pipeline's own +work: State, Hooks and the call through `dyn Filter`. + +## Phase 1: the core crate + +**Done** (2026-10-27): `packages/codeupipe-rs`, 47 tests (16 Payload, 17 +Pipeline, 7 labels, 3 example, 4 doc tests, 3 of them `compile_fail`), +`cargo clippy -- -D warnings` and `cargo fmt --check` clean. The tests are +ported from `packages/codeupipe-ts/tests`, except those that have no Rust form: +async steps, two runs at the same time, a class with no name, a key named like +an Object property. + +The eight primitives, to the contract, with the decisions above. + +Steps: + +1. `value.rs` and `payload.rs`, with tests first (RED, GREEN, REFACTOR), from + the Payload table of the contract. +2. The traits: `Filter`, `Predicate`, `Tap`, `Hook`. `step_label.rs`. +3. `State`, `Valve`, `Pipeline` (`add_filter`, `add_valve`, `add_tap`, + `add_pipeline`, `use_hook`, `run`, `state`, `labels`, `observe`), and + `NamedPipeline`. +4. The pricing example, with its test, giving + `{"price": 90.0, "qty": 12.0, "bulk": true}`. +5. The README, in the form of the TypeScript README: the primitives, the + example, the names, the differences from Python. + +Done when: `cargo test`, `cargo clippy -- -D warnings` and `cargo fmt --check` +pass, and `cargo run --example pricing` prints the result and the State. + +## Phase 2: conformance + +**Done** (2026-10-27): `rust:run` passed all 27 cases, with nothing +unsupported. The contract has since added three cases, from "A State +belongs to one run" (`run(p, state)`, and a run inside a run); Rust does not +pass them yet; "A State for each run" below adds them. To check that the runner catches a broken runtime, the core was +changed so that it did not record a Tap failure: 3 cases failed. The runner +has 7 tests of its JSON reader and writer. `bench.py` measures `rust:run`, its +baseline is kept under its own key (`rust`, not a Python version), and CI has a `rust` job, which also +builds with Rust 1.75 (D9). + +One thing that only Rust needed: `add_filter` and `add_valve` take the type +of the step, which gives its label. A step that the runner picks at run time +cannot be a `Box` (its label would be `Box`), so the runner hands +each Filter over by its own type through a small generic trait (`UseFilter`). + +The first numbers showed `rust:run` 4 to 5 times faster than `python:run` +but about 2 times **slower** than compiled Python. The time was in +allocation, not in dispatch (50 no-op steps take 0.42 µs). Three changes to +the core, with no change to its behaviour or its API: + +1. **`Map::insert` looks the key up first**, so a key that is there makes no + new `String`. +2. **State keeps its storage between runs** and writes the labels over, so a + run like the last one makes no new label `String`. +3. **A key is an `Arc`, made once**, shared by the index and by every + copy of the map. A map of 8 keys or fewer is searched in order and has no + hash index; a larger one gets an index with room to grow. The hash stays + std's SipHash: a faster hasher with no random seed would let crafted keys + slow down a map that is read from untrusted JSON. + +µs per run, Apple Silicon, one run: + +| scenario | python:run | python:static-inline | rust:run before | rust:run now | +|---|---|---|---|---| +| `micro-pass` (10 filters) | 4.94 | 0.606 | 1.10 | 0.246 | +| `valve-gated` | 5.81 | 0.801 | 1.24 | 0.301 | +| `taps` | 5.34 | 0.602 | 1.17 | 0.256 | +| `hook` | 5.88 | — | 1.10 | 0.247 | +| `heavy-allocation` (50 inserts) | 16.05 | 1.49 | 7.26 | 4.24 | + +The dynamic Rust run is now 20 times faster than `python:run`, and 2.5 times +faster than compiled Python, except where a run makes 50 new keys: there each +new key is an allocation and a hash, and a Python dict is hard to beat. **So +phase 3 waits**: the dynamic run already costs about 25 ns a step, and a +compiler would remove nanoseconds that few pipelines would notice. The README +says "no compiler yet". + +A runner in `conformance/runners/rust/`, with the protocol of the others: +`runner < cases.json > results.json`. It never judges. + +- **Steps.** `cup_ops/` has one file for each step type, with the names that + the TypeScript runner uses: `SetFilter`, `CopyFilter`, `AddFilter`, + `AppendFilter`, `RemoveFilter`, `MergeFilter`, `HasFilter`, `MutateFilter`, + `FailFilter`, `RecordTap`, `FailTap`, `NoopTap`, `EqualsPredicate`, + `TruthyPredicate`, `RecordHook`, `NestedPipeline`. +- **JSON.** The runner has its own small JSON reader and writer. A runner must + not download anything, so `serde_json` is not used. The core crate does not + get JSON either: principle 5 says that core holds nothing that is not in the + contract. +- **`run.py`.** A `_rust(target)` entry runs + `cargo run --quiet --release --offline --manifest-path conformance/runners/rust/Cargo.toml -- `. + `default_targets()` leaves out `rust:*` when there is no `cargo`, as it + leaves out TypeScript when there is no Node 22.18. The target `rust:run` is + added to `TARGETS`. +- **Benchmark.** `rust:run` is added to `bench.py`. Its number, beside + `python:run` and the compiled Python forms, decides phase 3 (below). +- **CI.** A `rust` job in `ci.yml`: `cargo fmt --check`, `cargo clippy`, + `cargo test`, the example, and `python conformance/run.py rust:run`. +- **Documents.** A Rust column in the "Runtimes" table of API.md, a row in the + README's table of packages, and `rust:run` in MATRIX.md. + +Done when: `rust:run` passes all 27 cases, with nothing unsupported. + +Open questions 3 and 4 of the conformance README (`equals` and `truthy` +across languages) get a Rust answer here. Recommendation: `equals` compares +`Value` by content, and `truthy` follows Python (an empty list or map is +false). No case depends on them yet. + +## A State for each run + +**Done** (2026-10-27). The contract on beta says that a State belongs to one +run, not to the Pipeline (conformance/API.md, "State"). In Rust: + +- **`run(&self, ...)`** in place of `run(&mut self, ...)`, so that a step can + hold the pipeline that it is in (an `Rc`, or a `Weak` to it) and run it + again. The steps were `&self` already. D8 changes with it. +- **`run_with_state(&input, &mut state)`** for the State from the caller; + Rust has no optional argument. `State::new()` makes one. +- **`state()` gives an `Rc`**: the State of the run that started last, + once it has ended. Each run takes a number when it starts, and its State + is kept at the end only if no later run has started. A snapshot cannot + change under a reader, and holding one never blocks a run. The cost: a + value borrowed from it needs the `Rc` in a variable first. +- **Errors in a State are `Arc`** (`SharedError`), so that a State + can be cloned: with the caller's State, the Pipeline keeps a copy for + `state()`. +- **Speed.** Two States take turns, the one that `state()` shows and the one + that the next run fills, so a run still makes no new label String. The + `Rc` of each run costs about 0.03 µs (10 filters: 0.25 to 0.28 µs). + +`rust:run` passes all 30 cases. Breaking the "started last" rule on purpose +fails one case and one unit test. The runner plays `run_again` as the +TypeScript runner does: a function, set for each case, that runs the +pipeline of the case again with a new State. + +## Phase 3: compile + +`codeupipe-compile` for Rust: it reads the source of a named pipeline and +writes one plain Rust module with no use of codeupipe. Same rule as the +others: **code that compiles gives the same output as the pipeline; a shape +that cannot keep that rule is refused, with the place named.** It is a +codeupipe pipeline itself, as the TypeScript compiler is: + +``` +ResolvePipelineFilter the add_steps body to a list of steps (a nested pipeline flattened) +LowerStepsFilter each step to a plain function +EmitModuleFilter the functions to one module +``` + +It depends on `syn` (to read Rust) and `prettyplease` (to write it), as the +TypeScript compiler depends on `typescript`. The core crate stays without +dependencies. + +```bash +codeupipe-compile src/pricing_pipeline.rs:PricingPipeline --out src/compiled/pricing.rs +``` + +Exit codes as in TypeScript: 0 written, 1 refused, 2 could not run. + +### Before it starts: is it worth it? + +In Python the compiled code is 6 to 8 times faster than the dynamic run, +because it removes the interpreter's work. Rust's dynamic run is already +machine code. What a compile removes in Rust is the State, the label strings, +the Hook loop and the call through `dyn Filter`. That could be 2 to 3 times, +or less. **Phase 2 measures `rust:run` first. Phase 3 starts only if the +number leaves room for a gain that is worth a compiler.** If it does not, the +compiler waits, and the Rust row in the README says "no compiler". + +Measured in phase 2 (2026-10-27): it does not, for now. After the work on the +core, a dynamic step costs about 25 ns, and the dynamic run is faster than +compiled Python in four of the five scenarios. The compiler waits until a +real pipeline shows a need. + +### The value type in compiled code + +Plain Python has `dict`, and plain TypeScript has `Map`. Plain Rust has no +JSON-shaped value. So the compiled module must carry one: a small `Value` enum +and its map, written into the module (about 60 lines). It is not an import of +codeupipe, so the rule holds, but it is code that the compiler writes and +that must stay the same as the core's `Value`. A test checks that the two +behave the same. + +### What compiles and what is refused + +The tables of the TypeScript compiler, in Rust: + +| Shape | Plain form | +|---|---| +| A named pipeline (`impl NamedPipeline`) | `pub fn run(input: &Map) -> Result`. `add_steps` holds only `add_*` calls | +| Filter (`call`) | A function on `&mut Map` | +| Valve | `if predicate(&data) { filter(&mut data)?; }` | +| Tap (`observe`) | The call, and its `Err` dropped: a failure does not stop the run | +| Nested pipeline | Its steps, in place | +| A setting of an instance (`self.rate`) | A `const`, from the literal that `DiscountFilter::new(0.1)` gives | +| `get`, `has`, `insert`, `remove`, `merge`, `to_dict` | Operations on the map | + +| Refused | What to do | +|---|---| +| A Hook, or `observe(...)` on the pipeline | Run the pipeline itself | +| An `add_steps` that does more than add steps | Keep only the `add_*` calls | +| An argument that is not a literal | Give literal settings | +| A step that calls its own helper method or another impl | Put the code in the one method | +| `copy`, `with_trace`, `trace_id`, `lineage` | Run the pipeline itself | +| The Payload given to other code | Use it only through its operations | +| A step type from another crate | Compile steps whose source is in the project | + +The fast mode only (no State, no Hooks), as in TypeScript. Python's +observable mode and inline layout come later, if the benchmark asks for them. + +### The `rust:static` target + +The TypeScript runner writes each case as a source file, compiles it and +imports it. Rust cannot load code while it runs, and a `cargo build` for each +case would take minutes. So the static target works in two stages: write +every case's pipeline as source, compile each into one generated crate (one +module for each case), build that crate once, and run it. + +Done when: `rust:static` passes 11 cases and the 16 State and Hook cases as ✅\*, +as `typescript:static` does, and the refusal tests pass. + +## Phase 4: the linter + +**Done** (2026-10-27): `packages/codeupipe-lint-rs/`, the crate +`codeupipe-lint` 0.1.0, with its own README. It is a codeupipe pipeline of +eleven Filters that carry a `LintRun` (a type of its own, D11), on `syn` +(`full`, `visit`) and `serde_json` for `cup.config.json`. The rules are +CUP-I000, A003, A005, I003, I005, I006, I007, and the warnings I008 and +I009; the arguments, the config file, the markers and the report are those +of the TypeScript linter. Decided on the way: + +- **The file name is snake_case** (`discount_filter.rs`), and a primitive is + a type that implements a primitive trait in the same file. +- **CUP-I008 is reported for each `impl`** of a type that is not a + primitive, not once for the type: in Rust the code of a type is in its + `impl` blocks, so the marker goes next to the code. Each `impl` of a + primitive (a helper method of a Filter, its `new`) is the primitive's code. +- **Comments are found between the tokens**, since a syntax tree has none. + Text in a doc comment or a string is not a marker. +- **A name is followed to its `let`** with the scopes of Rust: a later `let` + replaces it, a closure sees the function around it, a nested function does + not, and the types declared in a block are seen in all of it. + +CI runs it with `--strict` on all the Rust code of the repository (the +runtime, its examples and tests, the conformance runner, and the linter), as +the Python linter runs. That needed 45 `cup: allow-plain` markers (the +runtime's own types, the runner's JSON reader and entry, the linter's +visitors) and a fix in the runtime's tests: six step types declared in a +test function moved to the top of the file, and the test of a plain nested +Pipeline's label says why with `cup: allow-anonymous`. + +What follows is the plan as it was written. + + +Rust's types already check several rules: + +| Rule | In Rust | +|---|---| +| `CUP-I004` a write outside a Filter | A type error (D2) | +| `CUP-A001`, `CUP-A002` a function as a step, `CUP-A008` a function as a Predicate | A type error (D5) | +| `CUP-A009` an object literal as a step | Not possible in Rust | + +The rules that remain need a linter: `CUP-A003` (a step type declared inside +a function), `CUP-A005` (a plain nested `Pipeline`), `CUP-I003` (State read +by a step), `CUP-I005` (one primitive in each file), `CUP-I006` (the file +name), `CUP-I007` (the role suffix), `CUP-I008` (logic outside a primitive), +`CUP-I009` (a marker that allows nothing). `codeupipe-lint` for Rust is a +pipeline on `syn`, as the TypeScript linter is a pipeline on `typescript`. + +This phase is last. The type system covers the rules that matter most for +correctness. The others are about form. + +## Order and size + +| Phase | What | Rough size | +|---|---|---| +| 0 | Settle D1 to D10 | One review of this page | +| 1 | Core crate, example, README | About 600 lines with tests (TypeScript: 415 lines of source) | +| 2 | Runner, `run.py`, bench, CI, documents | About 400 lines, most of it the JSON reader and the `cup_ops` | +| 3 | Compiler and `rust:static`, if the benchmark says so | About 1,000 lines (TypeScript: 851 lines of source) | +| 4 | Linter. **Done** | About 1,200 lines (written: 2,060 of source with its comments, 950 of tests) | + +Phases 1 and 2 are one release: `codeupipe` 0.1.0 for Rust, in the +conformance matrix. Phases 3 and 4 are later versions. diff --git a/examples/pricing/__init__.py b/examples/pricing/__init__.py index 6a76565..9dbfefc 100644 --- a/examples/pricing/__init__.py +++ b/examples/pricing/__init__.py @@ -3,7 +3,7 @@ discount_filter.py Filter takes the Payload, writes to it, returns it mark_bulk_filter.py Filter at_least_predicate.py Predicate a yes-or-no question; the pipeline joins it to a Filter with add_valve - audit_tap.py Tap reads the Payload; its failure never halts the run + audit_tap.py Tap reads the Payload; if it fails, State records it and the run goes on print_steps_hook.py Hook sees each step start pricing_pipeline.py Pipeline where they come together @@ -14,5 +14,5 @@ used again in another pipeline. Run it: python -m examples.pricing -Export it: python -m codeupipe.lower examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing +Compile it: python -m codeupipe.compile examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing """ diff --git a/examples/pricing/audit_tap.py b/examples/pricing/audit_tap.py index bc4846f..d195ca0 100644 --- a/examples/pricing/audit_tap.py +++ b/examples/pricing/audit_tap.py @@ -2,8 +2,8 @@ class AuditTap: - """Reads the Payload and returns nothing. This one fails every time, to - show that a Tap failure is recorded and never halts the run.""" + """Reads the Payload and returns nothing. If it fails, State records the + error and the run goes on.""" def observe(self, payload): - raise ConnectionError("audit sink is down") + print("audit:", payload.to_dict()) diff --git a/examples/pricing/pricing_pipeline.py b/examples/pricing/pricing_pipeline.py index 06442db..14b9afe 100644 --- a/examples/pricing/pricing_pipeline.py +++ b/examples/pricing/pricing_pipeline.py @@ -14,5 +14,5 @@ def __init__(self): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) diff --git a/examples/pricing/print_steps_hook.py b/examples/pricing/print_steps_hook.py index 099cb87..f8e20b8 100644 --- a/examples/pricing/print_steps_hook.py +++ b/examples/pricing/print_steps_hook.py @@ -4,12 +4,12 @@ class PrintStepsHook(Hook): - """Sees the lifecycle. ``step`` is the step's identity; it is None for - the pipeline itself.""" + """Sees the lifecycle. ``step`` is the label of the step, which is the + name of its class. It is None for the pipeline itself.""" def before(self, step, payload): if step is not None: - print(" " + step.replace("examples.pricing.", "")) + print(" " + step) def after(self, step, payload): pass diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..727adf1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,587 @@ +{ + "name": "codeupipe-repository", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "codeupipe-repository", + "workspaces": [ + "packages/codeupipe-ts", + "packages/codeupipe-lint-ts", + "packages/codeupipe-compile-ts" + ] + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/codeupipe": { + "resolved": "packages/codeupipe-ts", + "link": true + }, + "node_modules/codeupipe-compile": { + "resolved": "packages/codeupipe-compile-ts", + "link": true + }, + "node_modules/codeupipe-lint": { + "resolved": "packages/codeupipe-lint-ts", + "link": true + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "packages/codeupipe-compile-ts": { + "name": "codeupipe-compile", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "codeupipe": "0.1.0", + "typescript": "^5.9.0" + }, + "bin": { + "codeupipe-compile": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^24.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "packages/codeupipe-lint-ts": { + "name": "codeupipe-lint", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "codeupipe": "0.1.0", + "typescript": "^5.9.0" + }, + "bin": { + "codeupipe-lint": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^24.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "packages/codeupipe-ts": { + "name": "codeupipe", + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@types/node": "^24.0.0", + "esbuild": "^0.28.2", + "typescript": "^5.9.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..a9ebf4f --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "codeupipe-repository", + "private": true, + "description": "The npm workspace of the TypeScript packages. It is not a package.", + "workspaces": [ + "packages/codeupipe-ts", + "packages/codeupipe-lint-ts", + "packages/codeupipe-compile-ts" + ], + "scripts": { + "build": "npm run build --workspaces", + "test": "npm test --workspaces", + "typecheck": "npm run typecheck --workspaces" + } +} diff --git a/packages/codeupipe-compile-ts/.gitignore b/packages/codeupipe-compile-ts/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/packages/codeupipe-compile-ts/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/packages/codeupipe-compile-ts/LICENSE b/packages/codeupipe-compile-ts/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-compile-ts/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-compile-ts/NOTICE b/packages/codeupipe-compile-ts/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-compile-ts/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-compile-ts/README.md b/packages/codeupipe-compile-ts/README.md new file mode 100644 index 0000000..ed4c08b --- /dev/null +++ b/packages/codeupipe-compile-ts/README.md @@ -0,0 +1,125 @@ +# codeupipe-compile for TypeScript + +Writes a codeupipe pipeline as one plain TypeScript module with no import +of codeupipe. It reads the source and does not run it. + +**The rule: code that compiles gives the same output as the pipeline. A +pipeline for which the compiler cannot keep that rule is refused, with the +place named.** A pipeline that compiles and then behaves in a different way +is a bug. + +The compiler is a codeupipe pipeline: +[src/compile-pipeline.ts](src/compile-pipeline.ts). It depends on +`codeupipe` and on the `typescript` package, which it uses to read and +write code. + +It is not on npm yet: the [README of the core package](../codeupipe-ts/README.md) +says how to install it from this repository. + +```bash +npx codeupipe-compile src/pricing-pipeline.ts:PricingPipeline --out build/pricing.ts +``` + +Exit code: **0** the module is written, **1** the pipeline is refused (the +message names the place and the fix), **2** the command could not run. +Nothing is written on 1 or 2. + +From TypeScript: `await compile(file, className, { outFile })` returns the +text of the module, and throws `CompileRefusal`. + +## Example + +The pricing example of the core package compiles to this module: + +```ts +const cup_01_discount_filter_rate: number = 0.1; +function cup_01_discount_filter(payload: Map) { + const price = cup_get(payload, "price", 0) * (1 - cup_01_discount_filter_rate); + return payload.set("price", Math.round(price * 100) / 100); +} + +// ... one function for each step ... + +export async function run(input: Record): Promise> { + const data = new Map(Object.entries(input)); + cup_01_discount_filter(data); + if (cup_02_when_at_least_predicate(data)) cup_02_mark_bulk_filter(data); + try { cup_03_audit_tap(data); } catch { /* a Tap failure does not stop the run */ } + return Object.fromEntries(data); +} +``` + +`run` takes a plain object and returns a new plain object. The object that +the caller gives is not changed. + +## What compiles + +| Shape | Plain form | +|---|---| +| A pipeline class | `export async function run(input)`. The class extends `Pipeline`, and its constructor calls `super()` and then only adds steps | +| Filter (`call`) | A function on a `Map`. An `async call` is awaited | +| Valve | `if (predicate(data)) filter(data);` | +| Tap (`observe`) | The call in `try`/`catch`: a failure does not stop the run | +| Nested pipeline | Its steps, in place | +| A setting of an instance (`this.rate`) | A constant, from the literal that `new DiscountFilter(0.1)` gives | +| `get`, `has`, `insert`, `remove`, `toDict`, `merge(new Payload(map))` | Operations on the `Map` | +| An import of a step file | The same import, relative to the compiled file | + +A step is made in the call that adds it: `this.addFilter(new X(...))`. The +arguments are literals: text, numbers, `true`, `false`, `null`, and lists +and objects of them. + +## What is refused + +| Shape | What to do | +|---|---| +| A Hook, or `observe(...)` on the pipeline | The compiled code has no lifecycle and no State. Run the pipeline itself | +| A constructor of the pipeline that does more than add steps | Keep only `super()` and the `add...` calls | +| An argument that is not a literal, or a setting that the constructor computes | Give literal settings, and compute in the method | +| A step that assigns to `this`, calls its own helper method, or extends a class | Put the code of the step in one method of one class | +| A Filter that returns something that is not the Payload it was given | Return the payload, or a write on it | +| `copy`, `withTrace`, `traceId`, `lineage` | Plain data has none of these. Run the pipeline itself | +| The Payload under another name, or given to other code | Use it only through its operations | +| A Tap or a Predicate that writes, an async Predicate | Only a Filter writes | +| A name that the step file declares beside the class | Move it to its own file and import it | +| A step class from a package | Compile steps whose source is in the project | + +The tests for each refusal are in [tests/refusals.test.ts](tests/refusals.test.ts). + +## What the compiled code does not have + +- **State.** No `executed`, `skipped`, `errors` or `timings`. +- **Hooks.** A pipeline with a Hook is refused. +- **Tap errors.** They are contained, as in the pipeline, but they are not + recorded. +- **Labels.** A function has a name such as `cup_01_discount_filter`. + +So the conformance suite judges the target `typescript:static` on output, +on Taps and on an unchanged input: 11 cases pass, and the 16 cases that +also check State or Hooks pass on what the code can show. + +The compiled module is like one instance of the pipeline: a setting that is +a list or an object is one value for every run. + +## Differences from the Python compiler + +Python has `python -m codeupipe.compile`. It reads a pipeline object that is +in memory. This compiler reads the source of a pipeline class. Python also +has an inline layout and an observable mode (State and Hooks); this +compiler has neither. + +## Development + +The three TypeScript packages are one npm workspace. Run `npm ci` once, at +the root of the repository. Then, in this directory: + +```bash +npm test # builds the core package first, then runs the tests +npm run typecheck # also checks the conformance runner +npm run build # JavaScript and type files in dist/ +python ../../conformance/run.py typescript:static +``` + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-compile-ts/package.json b/packages/codeupipe-compile-ts/package.json new file mode 100644 index 0000000..53810e8 --- /dev/null +++ b/packages/codeupipe-compile-ts/package.json @@ -0,0 +1,53 @@ +{ + "name": "codeupipe-compile", + "version": "0.1.0", + "private": true, + "description": "Writes a codeupipe pipeline as plain TypeScript with no import of codeupipe.", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/orchestrate-solutions/codeupipe.git", + "directory": "packages/codeupipe-compile-ts" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "codeupipe-compile": "./dist/cli.js" + }, + "files": [ + "dist", + "LICENSE", + "NOTICE", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "core": "npm --prefix ../codeupipe-ts run build", + "pretest": "npm run core", + "test": "node --test \"tests/*.test.ts\"", + "pretypecheck": "npm run core", + "typecheck": "tsc --noEmit", + "prebuild": "npm run core", + "build": "tsc -p tsconfig.build.json", + "prepack": "npm run build" + }, + "dependencies": { + "codeupipe": "0.1.0", + "typescript": "^5.9.0" + }, + "devDependencies": { + "@types/node": "^24.0.0" + }, + "keywords": [ + "pipeline", + "filter", + "payload", + "composition" + ] +} diff --git a/packages/codeupipe-compile-ts/src/cli.ts b/packages/codeupipe-compile-ts/src/cli.ts new file mode 100644 index 0000000..820aabf --- /dev/null +++ b/packages/codeupipe-compile-ts/src/cli.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env node +/** + * codeupipe-compile FILE:ClassName --out FILE + * + * Exit code: 0 the module is written, 1 the pipeline is refused (the message + * names the place and the fix), 2 the command could not run. Nothing is + * written on 1 or 2. + */ +import { parseArgs } from "node:util"; + +import { compile } from "./compile.ts"; +import { CompileRefusal } from "./compile-refusal.ts"; + +async function main(argv: string[]): Promise { + let target: string | undefined; + let outFile: string | undefined; + try { + const args = parseArgs({ args: argv, allowPositionals: true, options: { out: { type: "string" } } }); + target = args.positionals[0]; + outFile = args.values.out; + if (args.positionals.length !== 1) target = undefined; + } catch (error) { + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + const colon = target?.lastIndexOf(":") ?? -1; + if (target === undefined || outFile === undefined || colon <= 0 || colon === target.length - 1) { + console.error("usage: codeupipe-compile FILE:ClassName --out FILE"); + return 2; + } + try { + await compile(target.slice(0, colon), target.slice(colon + 1), { outFile }); + } catch (error) { + if (error instanceof CompileRefusal) { + console.error(`refused: ${error.message}`); + return 1; + } + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + console.log(outFile); + return 0; +} + +process.exitCode = await main(process.argv.slice(2)); diff --git a/packages/codeupipe-compile-ts/src/compile-pipeline.ts b/packages/codeupipe-compile-ts/src/compile-pipeline.ts new file mode 100644 index 0000000..c47a806 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile-pipeline.ts @@ -0,0 +1,15 @@ +import { Pipeline } from "codeupipe"; + +import { EmitModuleFilter } from "./emit-module-filter.ts"; +import { LowerStepsFilter } from "./lower-steps-filter.ts"; +import { ResolvePipelineFilter } from "./resolve-pipeline-filter.ts"; + +/** The compiler, as a codeupipe pipeline. */ +export class CompilePipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new ResolvePipelineFilter()); // the pipeline class to a list of steps + this.addFilter(new LowerStepsFilter()); // each step to a plain function + this.addFilter(new EmitModuleFilter()); // the functions to one module + } +} diff --git a/packages/codeupipe-compile-ts/src/compile-refusal.ts b/packages/codeupipe-compile-ts/src/compile-refusal.ts new file mode 100644 index 0000000..5f23d1d --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile-refusal.ts @@ -0,0 +1,18 @@ +/** + * The pipeline has a shape that the compiler does not write as plain code. + * + * The rule of the compiler: code that it writes behaves the same as the + * pipeline. A shape for which it cannot keep that rule is refused, with the + * place named and a hint that says what to do. + */ +export class CompileRefusal extends Error { + readonly where: string; + readonly hint: string; + + constructor(where: string, what: string, hint: string) { + super(`${where}: ${what}. ${hint}`); + this.name = "CompileRefusal"; + this.where = where; + this.hint = hint; + } +} diff --git a/packages/codeupipe-compile-ts/src/compile.ts b/packages/codeupipe-compile-ts/src/compile.ts new file mode 100644 index 0000000..e856276 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile.ts @@ -0,0 +1,28 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +import { Payload } from "codeupipe"; + +import { CompilePipeline } from "./compile-pipeline.ts"; +import { realPath } from "./real-path.ts"; +import { clearSources } from "./source-files.ts"; + +/** + * Write the pipeline class `className` of `file` as one plain TypeScript + * module, and return its text. With `outFile`, the module is written there + * and its imports are relative to that place. + * + * Throws CompileRefusal for a pipeline that has no plain form. Nothing is + * written then. + */ +export async function compile(file: string, className: string, options: { outFile?: string } = {}): Promise { + clearSources(); + const outFile = options.outFile === undefined ? undefined : realPath(options.outFile); + const out = await new CompilePipeline().run(new Payload({ file: resolve(file), className, outFile: outFile ?? null })); + const code = out.get("code") ?? ""; + if (outFile !== undefined) { + mkdirSync(dirname(outFile), { recursive: true }); + writeFileSync(outFile, code); + } + return code; +} diff --git a/packages/codeupipe-compile-ts/src/emit-module-filter.ts b/packages/codeupipe-compile-ts/src/emit-module-filter.ts new file mode 100644 index 0000000..4db7e2e --- /dev/null +++ b/packages/codeupipe-compile-ts/src/emit-module-filter.ts @@ -0,0 +1,123 @@ +import { basename, dirname, relative, sep } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { realPath } from "./real-path.ts"; +import { importsOf, isCoreImport } from "./source-files.ts"; +import type { ImportBinding } from "./source-files.ts"; +import type { LoweredFunction, LoweredStep } from "./step-plan.ts"; + +const HELPERS: Record = { + cup_get: `function cup_get(data: Map, key: string): T | null; +function cup_get(data: Map, key: string, fallback: T): T; +function cup_get(data: Map, key: string, fallback: unknown = null): unknown { + return data.has(key) ? data.get(key) : fallback; +}`, + cup_remove: `function cup_remove(data: Map, key: string): Map { + data.delete(key); + return data; +}`, + cup_merge: `function cup_merge(data: Map, other: Record): Map { + for (const [key, value] of Object.entries(other)) data.set(key, value); + return data; +}`, +}; + +/** The names that a file declares at its top level, without its imports. */ +function topLevelNames(tree: ts.SourceFile): Set { + const names = new Set(); + for (const statement of tree.statements) { + if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) names.add(declaration.name.getText()); + } else if ((ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement) || ts.isInterfaceDeclaration(statement) + || ts.isTypeAliasDeclaration(statement) || ts.isEnumDeclaration(statement)) && statement.name !== undefined) { + names.add(statement.name.text); + } + } + return names; +} + +/** + * Writes the module: the imports that the steps need, the helpers, one + * function for each step, and `run`. + * + * Input keys: `lowered`, `file`, `className`, optional `outFile` (the place + * of the module, so that its imports are relative to it). + * Output keys: `code`. + */ +export class EmitModuleFilter implements Filter { + call(payload: Payload): Payload { + const lowered = payload.get("lowered") ?? []; + const outFile = payload.get("outFile") ?? undefined; + const functions: LoweredFunction[] = lowered.flatMap((step) => + step.kind === "filter" ? [step.filter] : step.kind === "tap" ? [step.tap] : [step.predicate, step.filter]); + + const imports = new Map(); + const helpers = new Set(); + for (const lower of functions) { + const bindings = importsOf(lower.source); + const declared = topLevelNames(lower.source.tree); + for (const name of lower.names) { + if (HELPERS[name] !== undefined) { + helpers.add(name); + continue; + } + const binding = bindings.find((candidate) => candidate.local === name); + if (binding !== undefined) { + if (isCoreImport(binding)) { + throw new CompileRefusal(lower.name, `the step uses \`${name}\` from codeupipe`, + "Compiled code has no import of codeupipe. Use only the operations of the Payload"); + } + const other = imports.get(name); + if (other !== undefined && (other.imported !== binding.imported || (other.file ?? other.specifier) !== (binding.file ?? binding.specifier))) { + throw new CompileRefusal(lower.name, `two steps import different things with the name \`${name}\``, + "Give one of the imports another name"); + } + imports.set(name, binding); + } else if (declared.has(name)) { + throw new CompileRefusal(lower.name, `the step uses \`${name}\`, which its file declares beside the class`, + `Move \`${name}\` to its own file and import it`); + } + } + } + + const specifierOf = (binding: ImportBinding): string => { + if (binding.file === undefined) return binding.specifier; + if (outFile === undefined) return binding.file; + const path = relative(dirname(outFile), realPath(binding.file)).split(sep).join("/"); + return path.startsWith(".") ? path : "./" + path; + }; + const importLines = [...imports.values()].sort((a, b) => (a.local < b.local ? -1 : 1)).map((binding) => { + const type = binding.typeOnly ? "type " : ""; + const what = binding.imported === "default" ? binding.local + : binding.imported === "*" ? `* as ${binding.local}` + : binding.imported === binding.local ? `{ ${binding.local} }` : `{ ${binding.imported} as ${binding.local} }`; + return `import ${type}${what} from ${JSON.stringify(specifierOf(binding))};`; + }); + + const invoke = (lower: LoweredFunction) => `${lower.isAsync ? "await " : ""}${lower.name}(data)`; + const runLines = lowered.map((step) => { + if (step.kind === "filter") return ` ${invoke(step.filter)};`; + if (step.kind === "valve") return ` if (${step.predicate.name}(data)) ${invoke(step.filter)};`; + // A Tap failure never stops the run. + return ` try { ${invoke(step.tap)}; } catch { /* a Tap failure does not stop the run */ }`; + }); + + const file = payload.get("file") ?? ""; + const parts = [ + `// Made by codeupipe-compile from ${basename(file)}:${payload.get("className")}. Do not edit.\n` + + "// This file has no import of codeupipe. It gives the same output as the pipeline.", + ...(importLines.length > 0 ? [importLines.join("\n")] : []), + ...[...helpers].sort().map((name) => HELPERS[name] ?? ""), + ...functions.map((lower) => [...lower.consts, lower.code].join("\n")), + "/** Runs the steps in order on a copy of `input`, and returns the result. */\n" + + "export async function run(input: Record): Promise> {\n" + + " const data = new Map(Object.entries(input));\n" + + (runLines.length > 0 ? runLines.join("\n") + "\n" : "") + + " return Object.fromEntries(data);\n}", + ]; + return payload.insert("code", parts.join("\n\n") + "\n"); + } +} diff --git a/packages/codeupipe-compile-ts/src/index.ts b/packages/codeupipe-compile-ts/src/index.ts new file mode 100644 index 0000000..32a2bc6 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/index.ts @@ -0,0 +1,4 @@ +/** codeupipe-compile: writes a codeupipe pipeline as plain TypeScript with no import of codeupipe. */ +export { compile } from "./compile.ts"; +export { CompilePipeline } from "./compile-pipeline.ts"; +export { CompileRefusal } from "./compile-refusal.ts"; diff --git a/packages/codeupipe-compile-ts/src/literal.ts b/packages/codeupipe-compile-ts/src/literal.ts new file mode 100644 index 0000000..0e843f2 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/literal.ts @@ -0,0 +1,32 @@ +import ts from "typescript"; + +/** + * Whether an expression is a literal: text, a number, true, false, null, + * undefined, or a list or an object of literals. The compiler writes such a + * value into the code as a constant. + */ +export function isLiteral(node: ts.Expression): boolean { + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isSatisfiesExpression(node)) { + return isLiteral(node.expression); + } + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node) || ts.isNumericLiteral(node) + || ts.isBigIntLiteral(node)) { + return true; + } + if (node.kind === ts.SyntaxKind.TrueKeyword || node.kind === ts.SyntaxKind.FalseKeyword + || node.kind === ts.SyntaxKind.NullKeyword) { + return true; + } + if (ts.isIdentifier(node)) return node.text === "undefined"; + if (ts.isPrefixUnaryExpression(node)) { + return (node.operator === ts.SyntaxKind.MinusToken || node.operator === ts.SyntaxKind.PlusToken) + && ts.isNumericLiteral(node.operand); + } + if (ts.isArrayLiteralExpression(node)) return node.elements.every((element) => isLiteral(element)); + if (ts.isObjectLiteralExpression(node)) { + return node.properties.every((property) => ts.isPropertyAssignment(property) + && (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) || ts.isNumericLiteral(property.name)) + && isLiteral(property.initializer)); + } + return false; +} diff --git a/packages/codeupipe-compile-ts/src/lower-class.ts b/packages/codeupipe-compile-ts/src/lower-class.ts new file mode 100644 index 0000000..c77222d --- /dev/null +++ b/packages/codeupipe-compile-ts/src/lower-class.ts @@ -0,0 +1,277 @@ +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { isLiteral } from "./literal.ts"; +import type { LoweredFunction, Made } from "./step-plan.ts"; + +export type Role = "Filter" | "Predicate" | "Tap"; + +const METHOD: Record = { Filter: "call", Predicate: "test", Tap: "observe" }; +const WRITES = new Set(["insert", "remove", "merge"]); +const DATA = "Map"; + +/** `DiscountFilter` gives `discount_filter`. */ +export function snakeCase(className: string): string { + return className.replace(/(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/g, "_").toLowerCase(); +} + +function isThis(node: ts.Node): boolean { + return node.kind === ts.SyntaxKind.ThisKeyword; +} + +type Setting = { value: string; type: string | undefined }; + +/** + * The settings of one instance: each property of the class, with the value + * that this `new ClassName(...)` gives it. Each value is a literal. + */ +function settingsOf(made: Made, where: string): Map { + const settings = new Map(); + const node = made.ref.node; + const constructor = node.members.find(ts.isConstructorDeclaration); + + for (const member of node.members) { + if (!ts.isPropertyDeclaration(member)) continue; + if (!ts.isIdentifier(member.name)) { + throw new CompileRefusal(where, `the property \`${member.name.getText()}\` does not have a plain name`, "Give it a plain name"); + } + if (member.initializer !== undefined && !isLiteral(member.initializer)) { + throw new CompileRefusal(where, `the property \`${member.name.text}\` starts with a value that is not a literal`, + "Start it with a literal, or take the value from the Payload"); + } + settings.set(member.name.text, { value: member.initializer?.getText() ?? "undefined", type: member.type?.getText() }); + } + if (constructor === undefined) { + if (made.args.length > 0) throw new CompileRefusal(where, "the class has no constructor, but it is given arguments", "Remove the arguments"); + return settings; + } + + const given = new Map(); + constructor.parameters.forEach((parameter, index) => { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined || ts.getModifiers(parameter)?.length) { + throw new CompileRefusal(where, `the constructor parameter \`${parameter.getText()}\` is not a plain name`, + "Use plain parameters, and assign each one to a property"); + } + const argument = made.args[index]; + if (argument !== undefined && !isLiteral(argument)) { + throw new CompileRefusal(where, `the argument \`${argument.getText()}\` is not a literal`, + "Give literal settings (text, numbers, true, false, null, lists and objects of them)"); + } + if (argument === undefined && parameter.initializer !== undefined && !isLiteral(parameter.initializer)) { + throw new CompileRefusal(where, `the default of \`${parameter.name.text}\` is not a literal`, "Use a literal default"); + } + given.set(parameter.name.text, { + value: argument?.getText() ?? parameter.initializer?.getText() ?? "undefined", type: parameter.type?.getText(), + }); + }); + if (made.args.length > constructor.parameters.length) { + throw new CompileRefusal(where, "the class is given more arguments than its constructor has parameters", "Remove the extra arguments"); + } + + for (const statement of constructor.body?.statements ?? []) { + const assignment = ts.isExpressionStatement(statement) && ts.isBinaryExpression(statement.expression) + && statement.expression.operatorToken.kind === ts.SyntaxKind.EqualsToken ? statement.expression : undefined; + const target = assignment !== undefined && ts.isPropertyAccessExpression(assignment.left) && isThis(assignment.left.expression) + ? assignment.left.name.text : undefined; + if (assignment === undefined || target === undefined) { + throw new CompileRefusal(where, `the constructor does more than assign settings (\`${statement.getText().split("\n")[0]}\`)`, + "Keep only statements of the form this.name = parameter"); + } + const value = assignment.right; + const fromParameter = ts.isIdentifier(value) ? given.get(value.text) : undefined; + if (fromParameter === undefined && !isLiteral(value)) { + throw new CompileRefusal(where, `the constructor computes the setting \`${target}\``, + "Assign a parameter or a literal, and compute in the method"); + } + settings.set(target, { + value: fromParameter?.value ?? value.getText(), + type: settings.get(target)?.type ?? fromParameter?.type, + }); + } + return settings; +} + +/** + * One step class as one plain function. + * + * The function takes the data as a `Map`. Each operation of the Payload + * becomes an operation of the Map, and each `this.setting` becomes a + * constant. A shape with no such form is refused. + */ +export function lowerClass(made: Made, role: Role, name: string): LoweredFunction { + const node = made.ref.node; + const where = made.ref.name; + const source = made.ref.source; + const methodName = METHOD[role]; + + if ((node.heritageClauses ?? []).some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword)) { + throw new CompileRefusal(where, "the class extends another class", "Put the code of the step in one class"); + } + const method = node.members.find((member): member is ts.MethodDeclaration => + ts.isMethodDeclaration(member) && member.name.getText() === methodName); + if (method === undefined || method.body === undefined) { + throw new CompileRefusal(where, `the class has no ${methodName}(payload) method`, `A ${role} has ${methodName}(payload)`); + } + const isAsync = (ts.getModifiers(method) ?? []).some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword); + if (method.asteriskToken !== undefined) { + throw new CompileRefusal(where, `${methodName} is a generator`, "Make it a plain method"); + } + if (isAsync && role === "Predicate") { + throw new CompileRefusal(where, "test is async", "A Predicate gives true or false at once. Make test sync"); + } + const parameter = method.parameters[0]; + if (method.parameters.length !== 1 || parameter === undefined || !ts.isIdentifier(parameter.name)) { + throw new CompileRefusal(where, `${methodName} does not take one plain parameter`, `Write ${methodName}(payload)`); + } + const data = parameter.name.text; + const settings = settingsOf(made, where); + const used = new Set(); + const helpers = new Set(); + let returns = 0; + let depth = 0; // the number of functions inside the method that the visitor is in + + /** The parameter itself, or a chain of writes on it: `payload.insert(..).remove(..)`. */ + const isData = (expression: ts.Expression): boolean => { + if (ts.isIdentifier(expression)) return expression.text === data; + return ts.isCallExpression(expression) && ts.isPropertyAccessExpression(expression.expression) + && WRITES.has(expression.expression.name.text) && isData(expression.expression.expression); + }; + + const refuse = (what: string, hint: string): never => { + throw new CompileRefusal(where, what, hint); + }; + + const transformer: ts.TransformerFactory = (context) => { + const factory = context.factory; + const call = (helper: string, typeArguments: readonly ts.TypeNode[] | undefined, args: ts.Expression[]) => { + helpers.add(helper); + return factory.createCallExpression(factory.createIdentifier(helper), typeArguments, args); + }; + + const visit = (node: ts.Node): ts.Node => { + if (ts.isIdentifier(node) && node.text.startsWith("cup_")) { + return refuse(`the name \`${node.text}\` starts with cup_`, "The compiler uses names that start with cup_. Use another name"); + } + + // this.setting + if (ts.isPropertyAccessExpression(node) && isThis(node.expression)) { + const setting = node.name.text; + if (!settings.has(setting)) { + return refuse(`${methodName} uses \`this.${setting}\`, which is not a property with a literal value`, + "Use a setting of the instance, or a function that the file imports"); + } + const parent = node.parent; + const isWritten = (ts.isBinaryExpression(parent) && parent.left === node + && parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment) + || ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent); + if (isWritten) { + return refuse(`${methodName} assigns to \`this.${setting}\``, "A step keeps nothing between runs. Put the value in the Payload"); + } + used.add(setting); + return factory.createIdentifier(`${name}_${setting}`); + } + if (isThis(node)) { + return refuse(`${methodName} uses \`this\` for more than a setting`, "Read settings as this.name, and use no other member"); + } + + // payload.operation(...) + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) && isData(node.expression.expression)) { + const operation = node.expression.name.text; + const inner = node.expression.expression; + const receiver = ts.isIdentifier(inner) ? inner : (visit(inner) as ts.Expression); + const args = node.arguments.map((argument) => visit(argument) as ts.Expression); + if (WRITES.has(operation) && role !== "Filter") { + return refuse(`${methodName} writes to the Payload (\`${data}.${operation}\`)`, "Only a Filter writes. Put the write in a Filter"); + } + if (operation === "insert" && args.length === 2) { + return factory.createCallExpression(factory.createPropertyAccessExpression(receiver, "set"), undefined, args); + } + if (operation === "remove" && args.length === 1) return call("cup_remove", undefined, [receiver, ...args]); + if (operation === "get" && (args.length === 1 || args.length === 2)) { + return call("cup_get", node.typeArguments, [receiver, ...args]); + } + if (operation === "has" && args.length === 1) { + return factory.createCallExpression(factory.createPropertyAccessExpression(receiver, "has"), undefined, args); + } + if (operation === "toDict" && args.length === 0) { + return factory.createCallExpression(factory.createPropertyAccessExpression(factory.createIdentifier("Object"), "fromEntries"), + undefined, [receiver]); + } + if (operation === "merge" && node.arguments.length === 1) { + const other = node.arguments[0]; + if (other !== undefined && ts.isNewExpression(other) && other.expression.getText() === "Payload" + && (other.arguments?.length ?? 0) <= 1) { + const map = other.arguments?.[0]; + return call("cup_merge", undefined, [receiver, map === undefined ? factory.createObjectLiteralExpression() : (visit(map) as ts.Expression)]); + } + return refuse(`${methodName} merges a Payload that it did not make there`, "Write payload.merge(new Payload(map))"); + } + return refuse(`${methodName} uses \`${data}.${operation}\` in a form that has no plain code`, + "The operations are get, has, insert, remove, merge(new Payload(map)) and toDict"); + } + + // the payload itself, in any other place + if (ts.isIdentifier(node) && node.text === data) { + const parent = node.parent; + const isName = (ts.isPropertyAccessExpression(parent) && parent.name === node) + || (ts.isPropertyAssignment(parent) && parent.name === node) + || ((ts.isMethodDeclaration(parent) || ts.isPropertyDeclaration(parent)) && parent.name === node); + if (isName) return node; + if (ts.isReturnStatement(parent) && depth === 0) return node; + if (ts.isPropertyAccessExpression(parent) && parent.expression === node) { + return refuse(`${methodName} uses \`${data}.${parent.name.text}\``, + "The operations are get, has, insert, remove, merge(new Payload(map)) and toDict"); + } + return refuse(`${methodName} gives the Payload to other code, or gives it another name`, + "Use the Payload only through its operations, and return it"); + } + + if (ts.isReturnStatement(node) && depth === 0 && role === "Filter") { + returns += 1; + if (node.expression === undefined || !isData(node.expression)) { + return refuse("call returns something that is not the Payload it was given", + "Return the payload, or a write on it: return payload.insert(key, value)"); + } + } + + if (ts.isFunctionLike(node) || ts.isClassLike(node)) { + depth += 1; + const result = ts.visitEachChild(node, visit, context); + depth -= 1; + return result; + } + return ts.visitEachChild(node, visit, context); + }; + return (body) => ts.visitEachChild(body, visit, context); + }; + + const result = ts.transform(method.body, [transformer]); + const body = result.transformed[0]; + if (body === undefined) throw new CompileRefusal(where, "the method has no body", "Give it a body"); + const last = method.body.statements[method.body.statements.length - 1]; + if (role === "Filter" && returns === 0 && !(last !== undefined && ts.isThrowStatement(last))) { + refuse("call does not return the Payload", "End call with return payload"); + } + const printed = ts.createPrinter({ removeComments: false }).printNode(ts.EmitHint.Unspecified, body, source.tree); + result.dispose(); + + const returnType = role === "Predicate" ? ": boolean" : ""; + const code = `${isAsync ? "async " : ""}function ${name}(${data}: ${DATA})${returnType} ${printed}`; + const consts = [...used].map((setting) => { + const { value, type } = settings.get(setting) ?? { value: "undefined", type: undefined }; + return `const ${name}_${setting}${type === undefined ? "" : `: ${type}`} = ${value};`; + }); + + // The identifiers that the code uses, to find which imports it needs. + const names = new Set(helpers); + const scan = ts.createSourceFile("lowered.ts", [...consts, code].join("\n"), ts.ScriptTarget.Latest, true); + const collect = (inner: ts.Node) => { + if (ts.isIdentifier(inner) && !(ts.isPropertyAccessExpression(inner.parent) && inner.parent.name === inner)) { + names.add(inner.text); + } + ts.forEachChild(inner, collect); + }; + collect(scan); + + return { name, consts, code, isAsync, names, source }; +} diff --git a/packages/codeupipe-compile-ts/src/lower-steps-filter.ts b/packages/codeupipe-compile-ts/src/lower-steps-filter.ts new file mode 100644 index 0000000..614b9e8 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/lower-steps-filter.ts @@ -0,0 +1,36 @@ +import type { Filter, Payload } from "codeupipe"; + +import { lowerClass, snakeCase } from "./lower-class.ts"; +import type { LoweredStep, StepPlan } from "./step-plan.ts"; + +/** + * Writes each step as a plain function. + * + * A function has the name `cup__`. The number is the + * place of the step in the run, so two uses of one class have two names. + * + * Input keys: `steps`. + * Output keys: `lowered` (a list of LoweredStep). + */ +export class LowerStepsFilter implements Filter { + call(payload: Payload): Payload { + const steps = payload.get("steps") ?? []; + const width = Math.max(2, String(steps.length).length); + const lowered: LoweredStep[] = steps.map((step, index) => { + const number = String(index + 1).padStart(width, "0"); + const name = (prefix: string, className: string) => `cup_${number}_${prefix}${snakeCase(className)}`; + if (step.kind === "filter") { + return { kind: "filter", filter: lowerClass(step.filter, "Filter", name("", step.filter.ref.name)) }; + } + if (step.kind === "tap") { + return { kind: "tap", tap: lowerClass(step.tap, "Tap", name("", step.tap.ref.name)) }; + } + return { + kind: "valve", + predicate: lowerClass(step.predicate, "Predicate", name("when_", step.predicate.ref.name)), + filter: lowerClass(step.filter, "Filter", name("", step.filter.ref.name)), + }; + }); + return payload.insert("lowered", lowered); + } +} diff --git a/packages/codeupipe-compile-ts/src/real-path.ts b/packages/codeupipe-compile-ts/src/real-path.ts new file mode 100644 index 0000000..a39cbae --- /dev/null +++ b/packages/codeupipe-compile-ts/src/real-path.ts @@ -0,0 +1,16 @@ +import { existsSync, realpathSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; + +/** + * The path with each symbolic link followed, also for a path that is not + * there yet: the part that exists is followed, and the rest is added. + * + * A relative import is correct only between two such paths, because the + * module loader follows the links before it reads the import. + */ +export function realPath(path: string): string { + const absolute = resolve(path); + if (existsSync(absolute)) return realpathSync(absolute); + const parent = dirname(absolute); + return parent === absolute ? absolute : join(realPath(parent), basename(absolute)); +} diff --git a/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts b/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts new file mode 100644 index 0000000..90a0c5b --- /dev/null +++ b/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts @@ -0,0 +1,93 @@ +import { basename } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { findClass, loadSource } from "./source-files.ts"; +import type { ClassRef } from "./source-files.ts"; +import type { Made, StepPlan } from "./step-plan.ts"; + +function extendsPipeline(node: ts.ClassDeclaration): boolean { + return (node.heritageClauses ?? []).some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword + && clause.types.some((type) => type.expression.getText() === "Pipeline")); +} + +/** + * Reads a pipeline class and lists its steps. + * + * The class extends `Pipeline`. Its constructor calls `super()` and then + * only adds steps: `this.addFilter(new X(...))`, `this.addValve(new P(...), + * new F(...))`, `this.addTap(new T(...))`, `this.addPipeline(new Inner())`. + * A nested pipeline is flattened in place. + * + * Input keys: `file`, `className`. + * Output keys: `steps` (a list of StepPlan). + */ +export class ResolvePipelineFilter implements Filter { + call(payload: Payload): Payload { + const file = payload.get("file") ?? ""; + const className = payload.get("className") ?? ""; + const pipeline = findClass(loadSource(file), className, `${basename(file)}:${className}`); + return payload.insert("steps", this.stepsOf(pipeline, [])); + } + + private stepsOf(pipeline: ClassRef, outer: string[]): StepPlan[] { + const where = pipeline.name; + if (outer.includes(where)) { + throw new CompileRefusal(where, "the pipeline is nested in itself", "Remove the cycle"); + } + if (!extendsPipeline(pipeline.node)) { + throw new CompileRefusal(where, "the class does not extend Pipeline", "Give the name of a class that extends Pipeline"); + } + if (pipeline.node.members.some((member) => ts.isMethodDeclaration(member) && member.name.getText() === "run")) { + throw new CompileRefusal(where, "the class has its own run", "Remove it: the compiler writes the run of Pipeline"); + } + const made = (expression: ts.Expression | undefined, role: string): Made => { + if (expression === undefined || !ts.isNewExpression(expression) || !ts.isIdentifier(expression.expression)) { + throw new CompileRefusal(where, `the ${role} is not \`new ClassName(...)\``, + "Make each step in the constructor of the pipeline, in the call that adds it"); + } + const ref = findClass(pipeline.source, expression.expression.text, where); + return { ref, args: expression.arguments ?? [], argSource: pipeline.source }; + }; + + const steps: StepPlan[] = []; + const constructor = pipeline.node.members.find(ts.isConstructorDeclaration); + for (const statement of constructor?.body?.statements ?? []) { + const call = ts.isExpressionStatement(statement) && ts.isCallExpression(statement.expression) + ? statement.expression : undefined; + if (call !== undefined && call.expression.kind === ts.SyntaxKind.SuperKeyword) continue; + const method = call !== undefined && ts.isPropertyAccessExpression(call.expression) + && call.expression.expression.kind === ts.SyntaxKind.ThisKeyword ? call.expression.name.text : undefined; + if (call === undefined || method === undefined) { + throw new CompileRefusal(where, `the constructor has a statement that does not add a step (\`${statement.getText().split("\n")[0]}\`)`, + "Keep only super() and the calls that add steps"); + } + if (method === "addFilter") { + steps.push({ kind: "filter", filter: made(call.arguments[0], "Filter") }); + } else if (method === "addValve") { + steps.push({ kind: "valve", predicate: made(call.arguments[0], "Predicate"), filter: made(call.arguments[1], "Filter") }); + } else if (method === "addTap") { + steps.push({ kind: "tap", tap: made(call.arguments[0], "Tap") }); + } else if (method === "addPipeline") { + const inner = made(call.arguments[0], "nested pipeline"); + if (inner.args.length > 0) { + throw new CompileRefusal(where, `the nested pipeline \`${inner.ref.name}\` takes arguments`, + "Give the nested pipeline no arguments"); + } + steps.push(...this.stepsOf(inner.ref, [...outer, where])); + } else if (method === "useHook") { + throw new CompileRefusal(where, "the pipeline has a Hook", + "The compiled code has no lifecycle. Remove the Hook, or run the pipeline itself"); + } else if (method === "observe") { + throw new CompileRefusal(where, "the pipeline turns on timing or lineage", + "The compiled code has no State. Remove the call, or run the pipeline itself"); + } else { + throw new CompileRefusal(where, `the constructor calls \`this.${method}\``, + "Keep only super() and the calls that add steps"); + } + } + return steps; + } +} diff --git a/packages/codeupipe-compile-ts/src/source-files.ts b/packages/codeupipe-compile-ts/src/source-files.ts new file mode 100644 index 0000000..d3ac45e --- /dev/null +++ b/packages/codeupipe-compile-ts/src/source-files.ts @@ -0,0 +1,132 @@ +import { existsSync, readFileSync, statSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; + +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; + +/** One parsed file. */ +export type Source = { path: string; tree: ts.SourceFile }; + +/** A class, and the file that it is in. */ +export type ClassRef = { source: Source; node: ts.ClassDeclaration; name: string }; + +/** One name that a file imports. `imported` is the exported name, "default" or "*". */ +export type ImportBinding = { + local: string; + imported: string; + specifier: string; + /** The file, for a relative import. Undefined for a package. */ + file: string | undefined; + typeOnly: boolean; +}; + +const cache = new Map(); + +export function loadSource(path: string): Source { + const absolute = resolve(path); + let source = cache.get(absolute); + if (source === undefined) { + const text = readFileSync(absolute, "utf8"); + source = { path: absolute, tree: ts.createSourceFile(absolute, text, ts.ScriptTarget.Latest, true) }; + cache.set(absolute, source); + } + return source; +} + +/** Forget the files that were read. For a tool that compiles a file again after it changes. */ +export function clearSources(): void { + cache.clear(); +} + +function isFile(path: string): boolean { + return existsSync(path) && statSync(path).isFile(); +} + +/** The file of a relative import, or undefined for a package. */ +function resolveModule(from: string, specifier: string): string | undefined { + if (!specifier.startsWith(".") && !specifier.startsWith("/")) return undefined; + const base = resolve(dirname(from), specifier); + const candidates = [base, base.replace(/\.js$/, ".ts"), base.replace(/\.mjs$/, ".mts"), base + ".ts", base + ".tsx", + join(base, "index.ts")]; + return candidates.find(isFile); +} + +/** The names that a file imports. */ +export function importsOf(source: Source): ImportBinding[] { + const found: ImportBinding[] = []; + for (const statement of source.tree.statements) { + if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) continue; + const clause = statement.importClause; + if (clause === undefined) continue; + const specifier = statement.moduleSpecifier.text; + const file = resolveModule(source.path, specifier); + const base = { specifier, file }; + if (clause.name !== undefined) { + found.push({ ...base, local: clause.name.text, imported: "default", typeOnly: clause.isTypeOnly }); + } + const named = clause.namedBindings; + if (named !== undefined && ts.isNamespaceImport(named)) { + found.push({ ...base, local: named.name.text, imported: "*", typeOnly: clause.isTypeOnly }); + } else if (named !== undefined) { + for (const element of named.elements) { + found.push({ + ...base, local: element.name.text, imported: (element.propertyName ?? element.name).text, + typeOnly: clause.isTypeOnly || element.isTypeOnly, + }); + } + } + } + return found; +} + +/** + * Whether an import is of codeupipe itself: the package by its name, or the + * index file of its source (the directory that has `payload.ts` and + * `pipeline.ts`). + */ +export function isCoreImport(binding: ImportBinding): boolean { + if (binding.specifier === "codeupipe" || binding.specifier.startsWith("codeupipe/")) return true; + if (binding.file === undefined || !/^index\.[cm]?[jt]s$/.test(basename(binding.file))) return false; + const directory = dirname(binding.file); + return ["payload", "pipeline"].every((name) => isFile(join(directory, name + ".ts")) || isFile(join(directory, name + ".js"))); +} + +function hasModifier(node: ts.Node, kind: ts.SyntaxKind): boolean { + return ts.canHaveModifiers(node) && (ts.getModifiers(node) ?? []).some((modifier) => modifier.kind === kind); +} + +function topLevelClass(source: Source, name: string): ts.ClassDeclaration | undefined { + return source.tree.statements.find( + (statement): statement is ts.ClassDeclaration => ts.isClassDeclaration(statement) && statement.name?.text === name); +} + +/** + * The class that the name `name` means in the file `source`: a class of the + * file, or a class that the file imports from a file of the project. + */ +export function findClass(source: Source, name: string, where: string): ClassRef { + const own = topLevelClass(source, name); + if (own !== undefined) return { source, node: own, name }; + + const binding = importsOf(source).find((candidate) => candidate.local === name); + if (binding === undefined) { + throw new CompileRefusal(where, `\`${name}\` is not a class of ${basename(source.path)} and is not imported there`, + "Declare the class at the top level of a file, and import it"); + } + if (binding.file === undefined) { + throw new CompileRefusal(where, `\`${name}\` comes from the package "${binding.specifier}", and its source is not read`, + "Compile steps whose source is in the project"); + } + const target = loadSource(binding.file); + if (binding.imported === "default") { + const node = target.tree.statements.find((statement): statement is ts.ClassDeclaration => + ts.isClassDeclaration(statement) && hasModifier(statement, ts.SyntaxKind.DefaultKeyword)); + if (node !== undefined) return { source: target, node, name: node.name?.text ?? name }; + } else if (binding.imported !== "*") { + const node = topLevelClass(target, binding.imported); + if (node !== undefined) return { source: target, node, name: binding.imported }; + } + throw new CompileRefusal(where, `\`${name}\` is not a class at the top level of ${basename(binding.file)}`, + "Declare the class in the file that it is imported from"); +} diff --git a/packages/codeupipe-compile-ts/src/step-plan.ts b/packages/codeupipe-compile-ts/src/step-plan.ts new file mode 100644 index 0000000..1dfcefe --- /dev/null +++ b/packages/codeupipe-compile-ts/src/step-plan.ts @@ -0,0 +1,32 @@ +import type ts from "typescript"; + +import type { ClassRef, Source } from "./source-files.ts"; + +/** A step class, with the arguments that the pipeline gives to its constructor. */ +export type Made = { ref: ClassRef; args: readonly ts.Expression[]; argSource: Source }; + +/** One step of the pipeline, in the order of the run. A nested pipeline is already flattened. */ +export type StepPlan = + | { kind: "filter"; filter: Made } + | { kind: "valve"; predicate: Made; filter: Made } + | { kind: "tap"; tap: Made }; + +/** One step as plain code. */ +export type LoweredFunction = { + /** The name of the function. */ + name: string; + /** The constants that hold the settings of the instance. */ + consts: string[]; + /** The function. */ + code: string; + isAsync: boolean; + /** The identifiers that the constants and the function use. */ + names: Set; + /** The file that the class is in. */ + source: Source; +}; + +export type LoweredStep = + | { kind: "filter"; filter: LoweredFunction } + | { kind: "valve"; predicate: LoweredFunction; filter: LoweredFunction } + | { kind: "tap"; tap: LoweredFunction }; diff --git a/packages/codeupipe-compile-ts/tests/cli.test.ts b/packages/codeupipe-compile-ts/tests/cli.test.ts new file mode 100644 index 0000000..6ff2222 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/cli.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { PIPELINE_IMPORT, project } from "./compile-in.ts"; + +const CLI = join(import.meta.dirname, "../src/cli.ts"); +const STEP = `export class SetFilter { call(payload) { return payload.insert("a", 1); } }\n`; +const pipeline = (extra = "") => PIPELINE_IMPORT + `import { SetFilter } from "./set-filter.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new SetFilter()); ${extra} } } +`; + +function run(cwd: string, ...args: string[]) { + const done = spawnSync(process.execPath, [CLI, ...args], { cwd, encoding: "utf8" }); + return { code: done.status, out: done.stdout, err: done.stderr }; +} + +test("exit 0: the module is written, and its path is printed", () => { + const root = project({ "set-filter.ts": STEP, "pipeline.ts": pipeline() }); + const done = run(root, "pipeline.ts:MainPipeline", "--out", "build/main.ts"); + assert.equal(done.code, 0, done.err); + assert.equal(done.out.trim(), "build/main.ts"); + assert.match(readFileSync(join(root, "build/main.ts"), "utf8"), /export async function run/); +}); + +test("exit 1: a refusal names the place and the fix, and nothing is written", () => { + const root = project({ "set-filter.ts": STEP, "pipeline.ts": pipeline("this.useHook(null);") }); + const done = run(root, "pipeline.ts:MainPipeline", "--out", "build/main.ts"); + assert.equal(done.code, 1); + assert.match(done.err, /^refused: MainPipeline: the pipeline has a Hook\. /); + assert.equal(existsSync(join(root, "build")), false); +}); + +test("exit 2: wrong arguments, and a file that is not there", () => { + const root = project({}); + assert.equal(run(root).code, 2); + assert.equal(run(root, "pipeline.ts:MainPipeline").code, 2); + assert.equal(run(root, "pipeline.ts", "--out", "x.ts").code, 2); + assert.equal(run(root, "nothing.ts:MainPipeline", "--out", "x.ts").code, 2); +}); diff --git a/packages/codeupipe-compile-ts/tests/compile-in.ts b/packages/codeupipe-compile-ts/tests/compile-in.ts new file mode 100644 index 0000000..476e37b --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/compile-in.ts @@ -0,0 +1,26 @@ +// A helper for the tests: write source files to a new directory and compile one pipeline class. +import { mkdtempSync, realpathSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { compile } from "../src/index.ts"; + +export const PIPELINE_IMPORT = `import { Pipeline } from "codeupipe";\n`; + +export type Compiled = { code: string; outFile: string; run(input: Record): Promise> }; + +export function project(files: Record): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "cup-compile-"))); + for (const [name, text] of Object.entries(files)) writeFileSync(join(root, name), text); + return root; +} + +/** Compile `className` of `pipeline.ts` and import the module that was written. */ +export async function compileIn(files: Record, className = "MainPipeline"): Promise { + const root = project(files); + const outFile = join(root, "compiled.ts"); + const code = await compile(join(root, "pipeline.ts"), className, { outFile }); + const module: { run: Compiled["run"] } = await import(pathToFileURL(outFile).href); + return { code, outFile, run: module.run }; +} diff --git a/packages/codeupipe-compile-ts/tests/compile.test.ts b/packages/codeupipe-compile-ts/tests/compile.test.ts new file mode 100644 index 0000000..1b57403 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/compile.test.ts @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { join } from "node:path"; +import { test } from "node:test"; + +import ts from "typescript"; + +import { compile } from "../src/index.ts"; +import { PIPELINE_IMPORT, compileIn, project } from "./compile-in.ts"; + +const PRICING = join(import.meta.dirname, "../../codeupipe-ts/examples/pricing/pricing-pipeline.ts"); + +test("the pricing example compiles, and the code has no import", async () => { + const code = await compile(PRICING, "PricingPipeline"); + assert.doesNotMatch(code, /^import /m); + assert.doesNotMatch(code, /codeupipe"/); + assert.match(code, /export async function run\(/); +}); + +test("the compiled pricing example gives the same output as the pipeline", async () => { + const { PricingPipeline } = await import(PRICING); + const { Payload } = await import("../../codeupipe-ts/src/index.ts"); + const file = join(project({}), "pricing.ts"); + await compile(PRICING, "PricingPipeline", { outFile: file }); + const compiled: { run(input: Record): Promise> } = await import(file); + for (const input of [{ price: 100.0, qty: 12 }, { price: 19.99, qty: 2 }, {}, { qty: 10 }]) { + const given = structuredClone(input); + const dynamic = (await new PricingPipeline().run(new Payload(input))).toDict(); + assert.deepEqual(await compiled.run(given), dynamic); + assert.deepEqual(given, input); // the caller's input is not changed + } +}); + +test("the compiled code passes a strict type check", async () => { + // Every JavaScript host has console, so a Tap may log. Nothing else of a + // browser or of Node is given. + const dir = project({ "host.d.ts": "declare const console: { log(...data: unknown[]): void };\n" }); + const file = join(dir, "pricing.ts"); + await compile(PRICING, "PricingPipeline", { outFile: file }); + const program = ts.createProgram([file, join(dir, "host.d.ts")], { + strict: true, noEmit: true, target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, lib: ["lib.es2022.d.ts"], types: [], + }); + const problems = ts.getPreEmitDiagnostics(program).map((d) => ts.flattenDiagnosticMessageText(d.messageText, " ")); + assert.deepEqual(problems, []); +}); + +test("each operation of the Payload has a plain form", async () => { + const compiled = await compileIn({ + "ops-filter.ts": `import { Payload } from "codeupipe"; +export class OpsFilter { + readonly extra: Record; + constructor(extra: Record) { this.extra = extra; } + call(payload: Payload): Payload { + payload.insert("had", payload.has("a")).remove("a"); + payload.merge(new Payload(this.extra)); + payload.insert("copy", payload.toDict()); + payload.insert("missing", payload.get("zz")); + return payload.insert("n", payload.get("n", 0) + 1).insert("last", true); + } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { OpsFilter } from "./ops-filter.ts"; +export class MainPipeline extends Pipeline { + constructor() { super(); this.addFilter(new OpsFilter({ b: 2, n: 5 })); } +} +`, + }); + assert.deepEqual(await compiled.run({ a: 1 }), { had: true, b: 2, n: 6, copy: { had: true, b: 2, n: 5 }, missing: null, last: true }); +}); + +test("a Valve, a Tap that fails, a second use of a class, and a nested pipeline", async () => { + const compiled = await compileIn({ + "add-filter.ts": `export class AddFilter { + by: number; + constructor(by = 1) { this.by = by; } + call(payload) { return payload.insert("n", payload.get("n", 0) + this.by); } +} +`, + "is-on-predicate.ts": `export class IsOnPredicate { test(payload) { return payload.get("on") === true; } }\n`, + "fail-tap.ts": `export class FailTap { observe(payload) { throw new Error("sink is down"); } }\n`, + "pipeline.ts": PIPELINE_IMPORT + `import { AddFilter } from "./add-filter.ts"; +import { FailTap } from "./fail-tap.ts"; +import { IsOnPredicate } from "./is-on-predicate.ts"; +class InnerPipeline extends Pipeline { + constructor() { super(); this.addFilter(new AddFilter(10)); this.addTap(new FailTap()); } +} +export class MainPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new AddFilter()); + this.addValve(new IsOnPredicate(), new AddFilter(100)); + this.addPipeline(new InnerPipeline()); + } +} +`, + }); + assert.deepEqual(await compiled.run({ on: true }), { on: true, n: 111 }); + assert.deepEqual(await compiled.run({ on: false }), { on: false, n: 11 }); + assert.match(compiled.code, /cup_01_add_filter\(data\);\n if \(cup_02_when_is_on_predicate\(data\)\) cup_02_add_filter\(data\);\n cup_03_add_filter\(data\);/); +}); + +test("an async step is awaited, and a Filter failure is the same error", async () => { + const compiled = await compileIn({ + "later-filter.ts": `export class LaterFilter { + async call(payload) { await Promise.resolve(); return payload.insert("later", true); } +} +`, + "check-filter.ts": `export class CheckFilter { + call(payload) { if (payload.has("bad")) throw new RangeError("bad input"); return payload; } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { CheckFilter } from "./check-filter.ts"; +import { LaterFilter } from "./later-filter.ts"; +export class MainPipeline extends Pipeline { + constructor() { super(); this.addFilter(new LaterFilter()); this.addFilter(new CheckFilter()); } +} +`, + }); + assert.deepEqual(await compiled.run({}), { later: true }); + assert.match(compiled.code, /await cup_01_later_filter\(data\);/); + await assert.rejects(compiled.run({ bad: 1 }), { name: "RangeError", message: "bad input" }); +}); + +test("an import of a step file is kept, relative to the compiled file", async () => { + const compiled = await compileIn({ + "rates.ts": `export const RATES: Record = { gold: 0.2 };\nexport function round(value: number): number { return Math.round(value * 100) / 100; }\n`, + "rate-filter.ts": `import type { Payload } from "codeupipe"; +import { RATES, round as roundMoney } from "./rates.ts"; +export class RateFilter { + call(payload: Payload): Payload { + const rate = RATES[payload.get("tier", "")] ?? 0; + return payload.insert("price", roundMoney(payload.get("price", 0) * (1 - rate))); + } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { RateFilter } from "./rate-filter.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new RateFilter()); } } +`, + }); + assert.match(compiled.code, /^import \{ RATES \} from "\.\/rates\.ts";\nimport \{ round as roundMoney \} from "\.\/rates\.ts";$/m); + assert.deepEqual(await compiled.run({ tier: "gold", price: 10 }), { tier: "gold", price: 8 }); +}); diff --git a/packages/codeupipe-compile-ts/tests/refusals.test.ts b/packages/codeupipe-compile-ts/tests/refusals.test.ts new file mode 100644 index 0000000..b6f01e0 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/refusals.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { CompileRefusal, compile } from "../src/index.ts"; +import { PIPELINE_IMPORT, project } from "./compile-in.ts"; + +/** Compile a pipeline of one step, and return the refusal. */ +async function refusal(step: string, add: string, extra: Record = {}): Promise { + const root = project({ + "step.ts": step, + "pipeline.ts": PIPELINE_IMPORT + `import { Step } from "./step.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); ${add} } } +`, + ...extra, + }); + const outFile = join(root, "compiled.ts"); + try { + await compile(join(root, "pipeline.ts"), "MainPipeline", { outFile }); + } catch (error) { + assert.ok(error instanceof CompileRefusal, String(error)); + assert.equal(existsSync(outFile), false, "nothing is written when the pipeline is refused"); + return error; + } + throw new Error("the pipeline was not refused"); +} + +const FILTER = "this.addFilter(new Step());"; +const filter = (body: string) => `export class Step { call(payload) { ${body} } }\n`; + +const CASES: Array<[string, string, string, RegExp]> = [ + ["a Hook", filter("return payload;"), FILTER + " this.useHook(new Step());", /the pipeline has a Hook/], + ["timing", filter("return payload;"), FILTER + " this.observe();", /timing or lineage/], + ["other code in the constructor", filter("return payload;"), FILTER + " console.log(1);", /does not add a step/], + ["a step that is not made in the call", filter("return payload;"), "const step = new Step(); this.addFilter(step);", /does not add a step/], + ["an argument that is not a literal", `export class Step { key; constructor(key) { this.key = key; } call(payload) { return payload; } }\n`, + "this.addFilter(new Step(process.env.KEY));", /is not a literal/], + ["a setting that the constructor computes", `export class Step { key; constructor(key) { this.key = key + "!"; } call(payload) { return payload; } }\n`, + `this.addFilter(new Step("a"));`, /computes the setting `key`/], + ["a write to this", `export class Step { count = 0; call(payload) { this.count += 1; return payload; } }\n`, FILTER, /assigns to `this\.count`/], + ["a helper method", `export class Step { help() { return 1; } call(payload) { return payload.insert("a", this.help()); } }\n`, FILTER, + /`this\.help`, which is not a property/], + ["a class that extends another", `class Base {}\nexport class Step extends Base { call(payload) { return payload; } }\n`, FILTER, /extends another class/], + ["a return of another Payload", `import { Payload } from "codeupipe";\nexport class Step { call(payload) { return new Payload({}); } }\n`, FILTER, + /returns something that is not the Payload/], + ["no return", filter(`payload.insert("a", 1);`), FILTER, /does not return the Payload/], + ["copy", filter(`const mine = payload.copy(); return payload;`), FILTER, /`payload\.copy` in a form that has no plain code/], + ["the trace id", filter(`payload.insert("t", payload.traceId); return payload;`), FILTER, /uses `payload\.traceId`/], + ["the Payload under another name", filter(`const other = payload; return payload;`), FILTER, /gives it another name/], + ["the Payload given to other code", filter(`console.log(payload); return payload;`), FILTER, /gives the Payload to other code/], + ["a merge of a Payload from elsewhere", filter(`return payload.merge(globalThis.other);`), FILTER, /merges a Payload that it did not make/], + ["a name beside the class", `const RATE = 0.1;\nexport class Step { call(payload) { return payload.insert("r", RATE); } }\n`, FILTER, + /Move `RATE` to its own file and import it/], + ["a value from codeupipe", `import { State } from "codeupipe";\nexport class Step { call(payload) { return payload.insert("s", new State()); } }\n`, FILTER, + /uses `State` from codeupipe/], + ["a name that starts with cup_", filter(`const cup_x = 1; return payload;`), FILTER, /starts with cup_/], + ["a Tap that writes", `export class Step { observe(payload) { payload.insert("a", 1); } }\n`, "this.addTap(new Step());", /observe writes to the Payload/], + ["a Predicate that writes", `export class Step { test(payload) { payload.remove("a"); return true; } call(payload) { return payload; } }\n`, + "this.addValve(new Step(), new Step());", /test writes to the Payload/], + ["an async Predicate", `export class Step { async test(payload) { return true; } call(payload) { return payload; } }\n`, + "this.addValve(new Step(), new Step());", /test is async/], + ["a class with no call", `export class Step { observe(payload) {} }\n`, FILTER, /has no call\(payload\) method/], +]; + +for (const [name, step, add, message] of CASES) { + test(`refused: ${name}`, async () => { + const error = await refusal(step, add); + assert.match(error.message, message); + assert.ok(error.hint.length > 0); + }); +} + +test("refused: a step from a package", async () => { + const root = project({ + "pipeline.ts": PIPELINE_IMPORT + `import { Step } from "some-package"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new Step()); } } +`, + }); + await assert.rejects(compile(join(root, "pipeline.ts"), "MainPipeline"), /comes from the package "some-package"/); +}); + +test("refused: a class that is not a Pipeline, and a pipeline that is nested in itself", async () => { + const root = project({ + "pipeline.ts": PIPELINE_IMPORT + `export class Plain {} +export class LoopPipeline extends Pipeline { constructor() { super(); this.addPipeline(new LoopPipeline()); } } +`, + }); + await assert.rejects(compile(join(root, "pipeline.ts"), "Plain"), /does not extend Pipeline/); + await assert.rejects(compile(join(root, "pipeline.ts"), "LoopPipeline"), /nested in itself/); + await assert.rejects(compile(join(root, "pipeline.ts"), "Missing"), /`Missing` is not a class/); +}); diff --git a/packages/codeupipe-compile-ts/tsconfig.build.json b/packages/codeupipe-compile-ts/tsconfig.build.json new file mode 100644 index 0000000..7f9bdff --- /dev/null +++ b/packages/codeupipe-compile-ts/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src", + "outDir": "dist", + "declaration": true + }, + "include": ["src"] +} diff --git a/packages/codeupipe-compile-ts/tsconfig.json b/packages/codeupipe-compile-ts/tsconfig.json new file mode 100644 index 0000000..f419b27 --- /dev/null +++ b/packages/codeupipe-compile-ts/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src", "tests", "../../conformance/runners/typescript"] +} diff --git a/packages/codeupipe-lint-rs/Cargo.lock b/packages/codeupipe-lint-rs/Cargo.lock new file mode 100644 index 0000000..99a44fb --- /dev/null +++ b/packages/codeupipe-lint-rs/Cargo.lock @@ -0,0 +1,123 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "codeupipe" +version = "0.1.0" + +[[package]] +name = "codeupipe-lint" +version = "0.1.0" +dependencies = [ + "codeupipe", + "proc-macro2", + "serde_json", + "syn 2.0.119", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/packages/codeupipe-lint-rs/Cargo.toml b/packages/codeupipe-lint-rs/Cargo.toml new file mode 100644 index 0000000..bf0df07 --- /dev/null +++ b/packages/codeupipe-lint-rs/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "codeupipe-lint" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +description = "A linter for Rust code that uses codeupipe." +license = "Apache-2.0" +repository = "https://github.com/orchestrate-solutions/codeupipe" +readme = "README.md" +keywords = ["pipeline", "filter", "payload", "lint"] +# Not on crates.io yet. A version below 1.0 is not stable. +publish = false + +[[bin]] +name = "codeupipe-lint" +path = "src/main.rs" + +[dependencies] +codeupipe = { path = "../codeupipe-rs" } +# Lines and columns of the syntax tree, outside a procedural macro. +proc-macro2 = { version = "1.0.80", default-features = false, features = ["span-locations"] } +serde_json = "1.0.100" +syn = { version = "2.0.60", default-features = false, features = ["full", "parsing", "printing", "clone-impls", "visit"] } + +# A workspace of its own. The crates of the root workspace have no +# dependencies and build with no download; this one needs syn. +[workspace] diff --git a/packages/codeupipe-lint-rs/LICENSE b/packages/codeupipe-lint-rs/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-lint-rs/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-lint-rs/NOTICE b/packages/codeupipe-lint-rs/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-lint-rs/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-lint-rs/README.md b/packages/codeupipe-lint-rs/README.md new file mode 100644 index 0000000..a07ce94 --- /dev/null +++ b/packages/codeupipe-lint-rs/README.md @@ -0,0 +1,161 @@ +# codeupipe-lint for Rust + +A linter for Rust code that uses codeupipe. It reads the source and does +not run it. + +The linter is a codeupipe pipeline: [src/lint_pipeline.rs](src/lint_pipeline.rs). +Its steps carry a type of their own, `LintRun`, in place of a `Payload`, +because a run holds syntax trees. It depends on `codeupipe`, on `syn` to +parse the files, and on `serde_json` to read the config file. + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. It is not on crates.io yet. +Build it from this repository: + +```bash +cargo install --path packages/codeupipe-lint-rs +``` + +```bash +codeupipe-lint src # files or directories +codeupipe-lint --exclude '*/generated/*' src +codeupipe-lint --disable CUP-I005 src +codeupipe-lint --strict src # a warning fails the run too +codeupipe-lint --rules # list the rule ids +codeupipe-lint # paths come from cup.config.json +``` + +Each finding is one line, and the message names the fix: + +``` +path:line:col CUP-A005 unnamed-pipeline-step Pipeline is a plain `Pipeline::new()`. A nested pipeline must be (...) +``` + +Exit code: **0** no findings, **1** findings, **2** a path could not be +read or the arguments cannot be used. + +From Rust: `lint_paths(&paths, &LintOptions { exclude, disable, strict })` +returns the `LintRun` (`findings`, `suppressed`, `errors`, `report`, +`exit_code`). + +## Rules + +The ids are those of the Python and TypeScript linters. + +| id | slug | what | +|---|---|---| +| CUP-I000 | parse-error | the file has a syntax error | +| CUP-A003 | local-type-step | a step whose type is declared in a function | +| CUP-A005 | unnamed-pipeline-step | a plain `Pipeline::new()` nested as a step | +| CUP-I003 | state-read-in-step | a Filter (`call`) or a Predicate (`test`) reads the State of a pipeline | +| CUP-I005 | one-primitive-per-file | a second primitive at the top level of a file | +| CUP-I006 | primitive-file-name | the file name is not the type name in snake_case | +| CUP-I007 | primitive-role-suffix | the type name does not end with its role | +| CUP-I008 | logic-outside-primitive | **a warning.** A function, an `impl` of a type that is not a primitive, a trait, or a `const` has logic (`if`, `match`, a loop, `let ... else`) | +| CUP-I009 | marker-allows-nothing | **a warning.** A `cup: allow-...` comment that allowed no finding | + +A step is the argument of `add_filter`, `add_tap` or `add_pipeline` (on any +receiver), and the two arguments of `add_valve(predicate, filter)`. + +A type is a primitive when it implements `Filter`, `Tap`, `Predicate`, +`Hook` or `NamedPipeline`, in the same file. Its role is the trait +(`NamedPipeline` is the role `Pipeline`). + +## Rules that the compiler checks + +The types of the Rust runtime refuse these, so the linter has no rule for +them: + +| id | In Rust | +|---|---| +| CUP-A001, A002, A008 | A closure or a function is not a step and not a predicate: it does not implement the trait | +| CUP-A009 | An object literal does not exist in Rust | +| CUP-I004 | A Tap, a Predicate and a Hook get `&Payload`, so only a Filter can write | +| CUP-I001 | The data of a Payload is private | + +A `--disable` or a `lint.disable` with one of these ids is an error, as for +any id the linter does not have. + +## Logic that is not in a primitive (CUP-I008) + +A project that uses codeupipe writes its logic as Filters, Predicates, Taps, +Hooks and Pipelines. Logic in a plain function or a plain type works as +well, so nothing else shows the mistake. This rule shows it. + +- **It is a warning.** It is in the report and it is counted apart + (`No findings. 3 warnings.`). It does not make the exit code 1. With + `--strict`, a warning fails the run as a finding does. +- **Flagged:** a function, each `impl` of a type that is not a primitive + (an inherent `impl` or a trait such as `Display`), a trait with a default + method, and the first `const` or `static` whose value has logic. The + modules of a file are checked the same way. +- **Not flagged:** the code of a primitive (each `impl` of a type that + implements a primitive trait, so a helper method of a Filter is fine), + code with no logic, code for the tests only (`#[test]`, `#[cfg(test)]`), + and the files under `tests/`. +- **Not seen:** a loop that is a method call (`map`, `filter`, `for_each`), + `&&`, `||`, `?`, and code in a macro call. +- For logic that must be plain code (the edge to a file or a server, the + entry of a program), say why in a comment: `// cup: allow-plain `. + +## Turn a finding off + +- **One line.** A comment on the line of the finding, on the line of the + `add_...` call around it, or alone on the line directly above: + `// cup: allow-anonymous` for a CUP-A finding; + `// cup: allow-state-read`, `allow-shared-file`, `allow-name` or + `allow-plain` for a CUP-I finding. Any text can follow the marker. + A finding of a function or a type is on the line of its name or of its + `impl`, so the comment goes after the doc comment and the attributes: + + ```rust + /// Reads the config file. + #[inline] + // cup: allow-plain the edge of the program: it reads a file before the pipeline runs + pub fn read_config() -> Config { + ``` + + A comment that allows no finding is itself a finding (CUP-I009), so an + old one does not stay. Text in a doc comment or a string is not a marker. +- **Files.** `--exclude GLOB`, or `lint.exclude` in the config file. In a + glob, `*` is any text, a `/` too. +- **A rule, for the project.** `--disable RULE`, or `lint.disable` in the + config file. A rule id that does not exist is an error. + +## Config file + +`cup.config.json`, in the directory that the command runs from. The linter +reads the `lint` section and no other. + +```json +{"lint": {"paths": ["src"], "exclude": ["*/generated/*"], "disable": ["CUP-I005"]}} +``` + +The file is used only when no paths are given on the command line. A key in +`lint` that the linter does not read is an error. + +## Limits + +- A name is followed to the `let` that gave its value, in the same + function, and a later `let` of the same name replaces it. A parameter, a + pattern, a name from another file and the result of a call are not + followed, so a function that returns a step of a local type is not seen. +- A type is matched by its name in the file. The linter reads syntax only: + it does not resolve `use`, macros or types. +- The files under a `tests` directory are not checked by CUP-I005, I006, + I007 and I008. + +## Development + +The linter is a Cargo workspace of its own, apart from the root workspace, +so that the runtime builds with no download. In this directory: + +```bash +cargo test # the tests, and the linter on the Rust code of this repository +cargo clippy --all-targets -- -D warnings +cargo run -q -- --strict src tests ../codeupipe-rs/src ../codeupipe-rs/examples ../codeupipe-rs/tests ../../conformance/runners/rust/src +``` + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-lint-rs/src/analyze_plain_logic_filter.rs b/packages/codeupipe-lint-rs/src/analyze_plain_logic_filter.rs new file mode 100644 index 0000000..3bb43c9 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_plain_logic_filter.rs @@ -0,0 +1,125 @@ +use codeupipe::{Filter, StepError}; +use proc_macro2::Span; +use syn::spanned::Spanned; +use syn::visit::Visit; +use syn::Item; + +use crate::lint_run::LintRun; +use crate::logic::has_logic; +use crate::primitive_kind::{ + attributes_of, is_test_file, is_test_only, primitive_names, self_type_name, +}; +use crate::rules::{make_finding, LOGIC_OUTSIDE_PRIMITIVE}; +use crate::source::Source; + +const WHAT: &str = "has logic and is not a Filter, a Predicate, a Tap, a Hook or a Pipeline"; + +/// CUP-I008, a warning: logic that is not in a primitive. +/// +/// A project that uses codeupipe writes its logic as Filters, Predicates, +/// Taps, Hooks and Pipelines. Logic in a plain function or a plain type works +/// as well, so nothing else shows the mistake. This rule shows it. +/// +/// Logic is a decision or a loop: `if`, `match`, `for`, `while`, `loop`, +/// `let ... else`. Flagged: a function, an `impl` of a type that is not a +/// primitive (once for each `impl`), a trait with a default method, and the +/// first `const` or `static` whose value has logic. Modules of the file are +/// checked the same way. Not flagged: the code of a primitive (each `impl` +/// of a type that implements a primitive trait), code with no logic, code +/// for the tests only (`#[test]`, `#[cfg(test)]`), and the files of `tests/`. +/// +/// Reads `sources`. Writes `findings`. +pub struct AnalyzePlainLogicFilter; + +impl Filter for AnalyzePlainLogicFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for source in run + .sources + .iter() + .filter(|source| !is_test_file(&source.path)) + { + let mut primitives = Vec::new(); + primitive_names(&source.tree.items, &mut primitives); + let mut module_code = None; + let mut found = Vec::new(); + Self::in_items( + &source.tree.items, + &primitives, + &mut found, + &mut module_code, + ); + found + .extend(module_code.map(|at| (at, "The code at the top of this file".to_string()))); + for (at, what) in found { + run.findings.push(Self::finding(source, at, &what)); + } + } + Ok(()) + } +} + +impl AnalyzePlainLogicFilter { + fn finding(source: &Source, at: Span, what: &str) -> crate::finding::Finding { + make_finding( + source, + at, + LOGIC_OUTSIDE_PRIMITIVE, + &format!("{what} {WHAT}"), + None, + ) + } + + /// The items of a file or a module with logic that is not in a primitive. + fn in_items( + items: &[Item], + primitives: &[String], + found: &mut Vec<(Span, String)>, + module_code: &mut Option, + ) { + for item in items + .iter() + .filter(|item| !is_test_only(attributes_of(item))) + { + match item { + Item::Fn(function) => { + if has_logic(|v| v.visit_block(&function.block)) { + found.push(( + function.sig.ident.span(), + format!("Function `{}`", function.sig.ident), + )); + } + } + Item::Impl(block) => { + let name = self_type_name(&block.self_ty).unwrap_or_default(); + if !primitives.contains(&name) && has_logic(|v| v.visit_item_impl(block)) { + found.push((block.impl_token.span(), format!("Type `{name}`"))); + } + } + Item::Trait(definition) => { + if has_logic(|v| v.visit_item_trait(definition)) { + found.push(( + definition.ident.span(), + format!("Trait `{}`", definition.ident), + )); + } + } + Item::Const(constant) => { + if module_code.is_none() && has_logic(|v| v.visit_expr(&constant.expr)) { + *module_code = Some(constant.span()); + } + } + Item::Static(value) => { + if module_code.is_none() && has_logic(|v| v.visit_expr(&value.expr)) { + *module_code = Some(value.span()); + } + } + Item::Mod(module) => { + if let Some((_, inner)) = &module.content { + Self::in_items(inner, primitives, found, module_code); + } + } + _ => {} + } + } + } +} diff --git a/packages/codeupipe-lint-rs/src/analyze_primitive_names_filter.rs b/packages/codeupipe-lint-rs/src/analyze_primitive_names_filter.rs new file mode 100644 index 0000000..7043a69 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_primitive_names_filter.rs @@ -0,0 +1,81 @@ +use std::path::Path; + +use codeupipe::{Filter, StepError}; + +use crate::lint_run::LintRun; +use crate::primitive_kind::{is_test_file, primitives_in}; +use crate::rules::{make_finding, PRIMITIVE_FILE_NAME, PRIMITIVE_ROLE_SUFFIX}; + +/// CUP-I007: the type name of a primitive ends with its role +/// (`DiscountFilter`, `AuditTap`, `PrintStepsHook`, `AtLeastPredicate`, +/// `PricingPipeline`). +/// +/// CUP-I006: the file name is the type name in snake_case +/// (`discount_filter.rs`). +/// +/// Checked: the primitives at the top level of a file. Not checked: plain +/// types, and the files of `tests/`. +/// +/// Reads `sources`. Writes `findings`. +pub struct AnalyzePrimitiveNamesFilter; + +impl Filter for AnalyzePrimitiveNamesFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for source in run + .sources + .iter() + .filter(|source| !is_test_file(&source.path)) + { + let file = Path::new(&source.path) + .file_name() + .unwrap_or_default() + .to_string_lossy(); + let (stem, extension) = file.split_at(file.find('.').unwrap_or(file.len())); + for primitive in primitives_in(&source.tree.items) { + let role = primitive.kind.name(); + let mut wanted = primitive.name.clone(); + if !wanted.ends_with(role) { + wanted.push_str(role); + let what = format!( + "{role} `{}` does not say what it is. Name it `{wanted}`", + primitive.name + ); + let finding = + make_finding(source, primitive.at, PRIMITIVE_ROLE_SUFFIX, &what, None); + run.findings.push(finding); + } + let snake = Self::snake_case(&wanted); + if stem != snake { + let what = format!("`{wanted}` is in `{file}`. Put it in `{snake}{extension}`"); + let finding = + make_finding(source, primitive.at, PRIMITIVE_FILE_NAME, &what, None); + run.findings.push(finding); + } + } + } + Ok(()) + } +} + +impl AnalyzePrimitiveNamesFilter { + /// `ParseHTMLFilter` gives `parse_html_filter`. + pub fn snake_case(name: &str) -> String { + let chars: Vec = name.chars().collect(); + let mut out = String::new(); + for (index, c) in chars.iter().enumerate() { + let before = index.checked_sub(1).map(|i| chars[i]); + let after = chars.get(index + 1); + let word_starts = c.is_uppercase() + && before.is_some_and(|b| { + b.is_lowercase() + || b.is_ascii_digit() + || (b.is_uppercase() && after.is_some_and(|a| a.is_lowercase())) + }); + if word_starts { + out.push('_'); + } + out.extend(c.to_lowercase()); + } + out + } +} diff --git a/packages/codeupipe-lint-rs/src/analyze_primitives_per_file_filter.rs b/packages/codeupipe-lint-rs/src/analyze_primitives_per_file_filter.rs new file mode 100644 index 0000000..07b0160 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_primitives_per_file_filter.rs @@ -0,0 +1,41 @@ +use codeupipe::{Filter, StepError}; + +use crate::lint_run::LintRun; +use crate::primitive_kind::{is_test_file, primitives_in}; +use crate::rules::{make_finding, ONE_PRIMITIVE_PER_FILE}; + +/// CUP-I005: one primitive in each file. +/// +/// Flagged: each primitive at the top level of a file after the first. Not +/// flagged: plain types and functions beside the primitive, types in a +/// function or a module of the file, and the files of `tests/`. +/// +/// Reads `sources`. Writes `findings`. +pub struct AnalyzePrimitivesPerFileFilter; + +impl Filter for AnalyzePrimitivesPerFileFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for source in run + .sources + .iter() + .filter(|source| !is_test_file(&source.path)) + { + let primitives = primitives_in(&source.tree.items); + let Some((first, rest)) = primitives.split_first() else { + continue; + }; + for other in rest { + let what = format!( + "{} `{}` shares this file with {} `{}`", + other.kind.name(), + other.name, + first.kind.name(), + first.name + ); + let finding = make_finding(source, other.at, ONE_PRIMITIVE_PER_FILE, &what, None); + run.findings.push(finding); + } + } + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/analyze_stale_markers_filter.rs b/packages/codeupipe-lint-rs/src/analyze_stale_markers_filter.rs new file mode 100644 index 0000000..7a580e4 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_stale_markers_filter.rs @@ -0,0 +1,46 @@ +use codeupipe::{Filter, StepError}; + +use crate::finding::Finding; +use crate::lint_run::LintRun; +use crate::rules::{rule, MARKER_ALLOWS_NOTHING}; + +/// Finds each `cup: allow-...` comment that allowed no finding (CUP-I009, a +/// warning). Such a comment stays when the code that it was for is gone or +/// has moved, or when its kind has a spelling error. +/// +/// Only a comment line that starts with the marker counts, so text that +/// names a marker in a string, a doc comment or the middle of a sentence is +/// not reported. +/// +/// Reads `sources` and `used_markers` (from ApplySuppressionsFilter). Writes +/// `findings`. +pub struct AnalyzeStaleMarkersFilter; + +impl Filter for AnalyzeStaleMarkersFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + let stale = rule(MARKER_ALLOWS_NOTHING).expect("a rule"); + for source in &run.sources { + let unused = source.markers.iter().filter(|marker| { + !run.used_markers + .iter() + .any(|(path, line)| *path == source.path && *line == marker.line) + }); + for marker in unused { + run.findings.push(Finding { + path: source.path.clone(), + line: marker.line, + col: marker.col, + rule: MARKER_ALLOWS_NOTHING.to_string(), + slug: stale.slug.to_string(), + message: format!( + "The comment `cup: {}` allows nothing. {}", + marker.kind, stale.fix + ), + call_line: None, + warning: true, + }); + } + } + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/analyze_state_reads_filter.rs b/packages/codeupipe-lint-rs/src/analyze_state_reads_filter.rs new file mode 100644 index 0000000..ea7d7fb --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_state_reads_filter.rs @@ -0,0 +1,220 @@ +use std::collections::{HashMap, HashSet}; + +use codeupipe::{Filter, StepError}; +use syn::spanned::Spanned; +use syn::visit::{self, Visit}; +use syn::{Expr, GenericArgument, Member, PathArguments, Type}; + +use crate::finding::Finding; +use crate::lint_run::LintRun; +use crate::primitive_kind::{self_type_name, PrimitiveKind}; +use crate::rules::{make_finding, STATE_READ}; +use crate::source::{position, text_of, Source}; + +/// The methods of State that read it. +const STATE_MEMBERS: &[&str] = &["executed", "skipped", "errors", "timings"]; + +/// Types that hold a value of the type in them: a field of `Rc` holds a State. +const HOLDERS: &[&str] = &[ + "Rc", "Arc", "RefCell", "Cell", "Mutex", "RwLock", "Box", "Option", "Weak", +]; + +/// CUP-I003: a Filter (`call`) or a Predicate (`test`) that reads the State +/// of a pipeline. +/// +/// Flagged: `.state()` followed by a method of State (`executed`, +/// `skipped`, `errors`, `timings`); `.state()` when the name says +/// it is a pipeline (`self.pipeline.borrow().state()`); and a field of the +/// step that holds a State (`seen: Rc`). Not flagged: `self.state()` +/// (the step's own method), a State that the step makes, a Payload key named +/// "state", and anything in a Tap or a Hook. +/// +/// Reads `sources`. Writes `findings`. +pub struct AnalyzeStateReadsFilter; + +impl Filter for AnalyzeStateReadsFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for source in &run.sources { + let mut fields = StateFields::default(); + fields.visit_file(&source.tree); + let mut steps = StepBodies { + source, + fields: &fields.by_type, + findings: Vec::new(), + }; + steps.visit_file(&source.tree); + run.findings.append(&mut steps.findings); + } + Ok(()) + } +} + +/// The fields of each struct of a file that hold a State. +#[derive(Default)] +struct StateFields { + by_type: HashMap>, +} + +impl<'ast> Visit<'ast> for StateFields { + fn visit_item_struct(&mut self, item: &'ast syn::ItemStruct) { + let held: HashSet = item + .fields + .iter() + .filter(|field| holds_state(&field.ty)) + .filter_map(|field| field.ident.as_ref().map(ToString::to_string)) + .collect(); + self.by_type.insert(item.ident.to_string(), held); + visit::visit_item_struct(self, item); + } +} + +// cup: allow-plain used by the analysis Filter: it reads a type of the syntax tree +fn holds_state(ty: &Type) -> bool { + match ty { + Type::Reference(reference) => holds_state(&reference.elem), + Type::Path(path) => path.path.segments.last().is_some_and(|last| { + let held = || { + match &last.arguments { + PathArguments::AngleBracketed(arguments) => arguments.args.iter().any(|argument| { + matches!(argument, GenericArgument::Type(inner) if holds_state(inner)) + }), + _ => false, + } + }; + last.ident == "State" || (HOLDERS.iter().any(|holder| last.ident == holder) && held()) + }), + _ => false, + } +} + +/// Finds the State reads in each `call` of a Filter and `test` of a Predicate. +struct StepBodies<'a> { + source: &'a Source, + fields: &'a HashMap>, + findings: Vec, +} + +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +impl<'ast> Visit<'ast> for StepBodies<'_> { + fn visit_item_impl(&mut self, item: &'ast syn::ItemImpl) { + let kind = item + .trait_ + .as_ref() + .and_then(|(_, path, _)| PrimitiveKind::of_trait(path)); + let method = match kind { + Some(PrimitiveKind::Filter) => "call", + Some(PrimitiveKind::Predicate) => "test", + _ => return visit::visit_item_impl(self, item), + }; + let none = HashSet::new(); + let fields = self_type_name(&item.self_ty) + .and_then(|name| self.fields.get(&name)) + .unwrap_or(&none); + let mut reads = Reads { + source: self.source, + fields, + found: Vec::new(), + }; + for inner in &item.items { + if let syn::ImplItem::Fn(function) = inner { + if function.sig.ident == method { + reads.visit_block(&function.block); + } + } + } + self.findings.append(&mut reads.found); + } +} + +/// The State reads in the body of one step. +struct Reads<'a> { + source: &'a Source, + fields: &'a HashSet, + found: Vec, +} + +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +impl Reads<'_> { + fn flag(&mut self, read: &dyn Spanned) { + let span = read.span(); + let at = position(span); + if self + .found + .iter() + .any(|finding| (finding.line, finding.col) == at) + { + return; + } + let what = format!( + "Step reads pipeline State (`{}`)", + text_of(&self.source.text, span) + ); + self.found + .push(make_finding(self.source, span, STATE_READ, &what, None)); + } +} + +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +impl<'ast> Visit<'ast> for Reads<'_> { + fn visit_item(&mut self, _: &'ast syn::Item) { + // An item in the body has its own scope. + } + + fn visit_expr_method_call(&mut self, call: &'ast syn::ExprMethodCall) { + if STATE_MEMBERS.iter().any(|member| call.method == member) && is_state_call(&call.receiver) + { + self.flag(&call.receiver); + } else if is_state_call_on(call) && names_a_pipeline(&call.receiver) { + self.flag(call); + } + visit::visit_expr_method_call(self, call); + } + + fn visit_expr_field(&mut self, field: &'ast syn::ExprField) { + let held = match &field.member { + Member::Named(name) => is_self(&field.base) && self.fields.contains(&name.to_string()), + Member::Unnamed(_) => false, + }; + if held { + self.flag(field); + } + visit::visit_expr_field(self, field); + } +} + +/// `.state()`, where `` is not `self`. +fn is_state_call(value: &Expr) -> bool { + matches!(value, Expr::MethodCall(call) if is_state_call_on(call)) +} + +fn is_state_call_on(call: &syn::ExprMethodCall) -> bool { + call.method == "state" && call.args.is_empty() && !is_self(&call.receiver) +} + +fn is_self(value: &Expr) -> bool { + matches!(value, Expr::Path(path) if path.path.is_ident("self")) +} + +/// The name of `value`, past the method calls on it, says it is a pipeline: +/// `self.pipeline.borrow().upgrade()`, `inner_pipeline`. +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +fn names_a_pipeline(value: &Expr) -> bool { + let name = match value { + Expr::MethodCall(call) => return names_a_pipeline(&call.receiver), + Expr::Paren(inner) => return names_a_pipeline(&inner.expr), + Expr::Reference(inner) => return names_a_pipeline(&inner.expr), + Expr::Unary(inner) => return names_a_pipeline(&inner.expr), + Expr::Field(field) => match &field.member { + Member::Named(name) => name.to_string(), + Member::Unnamed(_) => return false, + }, + Expr::Path(path) => path + .path + .segments + .last() + .map(|s| s.ident.to_string()) + .unwrap_or_default(), + _ => return false, + }; + name.to_lowercase().contains("pipeline") +} diff --git a/packages/codeupipe-lint-rs/src/analyze_steps_filter.rs b/packages/codeupipe-lint-rs/src/analyze_steps_filter.rs new file mode 100644 index 0000000..e012dbf --- /dev/null +++ b/packages/codeupipe-lint-rs/src/analyze_steps_filter.rs @@ -0,0 +1,30 @@ +use codeupipe::{Filter, StepError}; +use syn::visit::Visit; + +use crate::lint_run::LintRun; +use crate::step_visitor::StepVisitor; + +/// CUP-A003 and CUP-A005: a step whose type is declared in a function, and +/// a plain `Pipeline` nested as a step. +/// +/// A step is the argument of `add_filter`, `add_tap` or `add_pipeline` (on +/// any receiver), or one of the two arguments of `add_valve(predicate, +/// filter)`. A name is followed to the `let` that gave its value, in the +/// same function. A parameter and a name from another file are not followed. +/// +/// A closure or a function as a step (CUP-A001, A002, A008) is a compile +/// error in Rust, so it has no rule here. +/// +/// Reads `sources`. Writes `findings`. +pub struct AnalyzeStepsFilter; + +impl Filter for AnalyzeStepsFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for source in &run.sources { + let mut visitor = StepVisitor::new(source); + visitor.visit_file(&source.tree); + run.findings.append(&mut visitor.findings); + } + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/apply_suppressions_filter.rs b/packages/codeupipe-lint-rs/src/apply_suppressions_filter.rs new file mode 100644 index 0000000..48a3470 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/apply_suppressions_filter.rs @@ -0,0 +1,71 @@ +use std::collections::HashMap; + +use codeupipe::{Filter, StepError}; + +use crate::lint_run::LintRun; +use crate::source::Marker; + +/// The markers that allow a CUP-A finding. +const ANONYMOUS: &[&str] = &["allow-anonymous"]; +/// The markers that allow a CUP-I finding. +const INVARIANT: &[&str] = &[ + "allow-state-read", + "allow-shared-file", + "allow-name", + "allow-plain", +]; + +/// Drops the findings that a comment allows. +/// +/// `// cup: allow-anonymous` allows a CUP-A finding. `// cup: allow-state-read`, +/// `allow-shared-file`, `allow-name` and `allow-plain` allow a CUP-I finding. +/// The comment is on the line of the finding, on the line of the `add_...` +/// call around it, or alone on the line directly above. Any text can follow +/// the marker. +/// +/// Reads `findings` and `sources`. Writes `findings` (kept), `suppressed` +/// (dropped), and `used_markers` (each comment that allowed a finding). +pub struct ApplySuppressionsFilter; + +impl Filter for ApplySuppressionsFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + let markers: HashMap<&str, &[Marker]> = run + .sources + .iter() + .map(|source| (source.path.as_str(), source.markers.as_slice())) + .collect(); + let mut kept = Vec::new(); + for finding in std::mem::take(&mut run.findings) { + let kinds = if finding.rule.starts_with("CUP-A") { + ANONYMOUS + } else { + INVARIANT + }; + let file = markers + .get(finding.path.as_str()) + .copied() + .unwrap_or_default(); + let allows = |marker: &&Marker| { + kinds.contains(&marker.kind.as_str()) + && (marker.line == finding.line + || Some(marker.line) == finding.call_line + || (marker.alone && marker.line + 1 == finding.line)) + }; + let places: Vec = file + .iter() + .filter(allows) + .map(|marker| marker.line) + .collect(); + if places.is_empty() { + kept.push(finding); + continue; + } + for line in places { + run.used_markers.push((finding.path.clone(), line)); + } + run.suppressed.push(finding); + } + run.findings = kept; + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/collect_files_filter.rs b/packages/codeupipe-lint-rs/src/collect_files_filter.rs new file mode 100644 index 0000000..803a62c --- /dev/null +++ b/packages/codeupipe-lint-rs/src/collect_files_filter.rs @@ -0,0 +1,63 @@ +use std::fs; +use std::path::Path; + +use codeupipe::{Filter, StepError}; + +use crate::fnmatch::fnmatch; +use crate::lint_run::LintRun; + +/// Directories that hold no source of the project: version control, build +/// output, and dependencies. +const SKIP_DIRS: &[&str] = &[".git", "target", "node_modules"]; + +/// The Rust files under the given paths. +/// +/// Reads `paths` (files or directories) and `exclude` (globs matched against +/// the path of each file). Writes `files`, and `errors` for a path that is +/// not there. Directories are walked in the order of their names. +pub struct CollectFilesFilter; + +impl Filter for CollectFilesFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + let mut found = Vec::new(); + for root in run.paths.clone() { + let path = Path::new(&root); + if path.is_dir() { + Self::walk(path, &mut found); + } else if path.is_file() { + found.push(root); + } else { + run.errors + .push(format!("{root}: no such file or directory")); + } + } + let exclude = &run.exclude; + run.files = found + .into_iter() + .filter(|file| file.ends_with(".rs")) + .filter(|file| !exclude.iter().any(|pattern| fnmatch(file, pattern))) + .collect(); + Ok(()) + } +} + +impl CollectFilesFilter { + /// The files under `directory`, in the order of their names. + fn walk(directory: &Path, found: &mut Vec) { + let Ok(entries) = fs::read_dir(directory) else { + return; + }; + let mut entries: Vec<_> = entries.flatten().map(|entry| entry.path()).collect(); + entries.sort(); + for path in entries { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + if path.is_dir() && !SKIP_DIRS.contains(&name.as_ref()) { + Self::walk(&path, found); + } else if path.is_file() { + // `./src/x.rs` is `src/x.rs`. + let path = path.strip_prefix(".").unwrap_or(&path); + found.push(path.to_string_lossy().into_owned()); + } + } + } +} diff --git a/packages/codeupipe-lint-rs/src/drop_disabled_filter.rs b/packages/codeupipe-lint-rs/src/drop_disabled_filter.rs new file mode 100644 index 0000000..589c84a --- /dev/null +++ b/packages/codeupipe-lint-rs/src/drop_disabled_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, StepError}; + +use crate::lint_run::LintRun; + +/// Drops the findings of the rules that the project switched off, from those +/// that are reported and from those that a comment allowed. +/// +/// Reads `disable`. Writes `findings` and `suppressed`. +pub struct DropDisabledFilter; + +impl Filter for DropDisabledFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + let disable = &run.disable; + run.findings + .retain(|finding| !disable.contains(&finding.rule)); + run.suppressed + .retain(|finding| !disable.contains(&finding.rule)); + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/finding.rs b/packages/codeupipe-lint-rs/src/finding.rs new file mode 100644 index 0000000..34435aa --- /dev/null +++ b/packages/codeupipe-lint-rs/src/finding.rs @@ -0,0 +1,28 @@ +/// One finding, at `path:line:col` (line and column start at 1). +/// +/// `call_line` is the line of the `add_...` call around the finding, when it +/// is a different line: a comment that suppresses can be on that line too. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Finding { + pub path: String, + pub line: usize, + pub col: usize, + pub rule: String, + pub slug: String, + pub message: String, + pub call_line: Option, + /// A warning is reported and does not fail the run, unless the run is strict. + pub warning: bool, +} + +// cup: allow-plain the text of one finding, for the report +impl Finding { + /// `path:line:col RULE slug message`, with `warning: ` before the message of a warning. + pub fn format(&self) -> String { + let warning = if self.warning { "warning: " } else { "" }; + format!( + "{}:{}:{} {} {} {}{}", + self.path, self.line, self.col, self.rule, self.slug, warning, self.message + ) + } +} diff --git a/packages/codeupipe-lint-rs/src/fnmatch.rs b/packages/codeupipe-lint-rs/src/fnmatch.rs new file mode 100644 index 0000000..b98d181 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/fnmatch.rs @@ -0,0 +1,68 @@ +/// Whether `path` matches `pattern`, with the rules of a shell glob as +/// Python's fnmatch has them: `*` is any text (a `/` too), `?` is one +/// character, `[abc]` is one of the characters, `[a-z]` a range, and `[!a]` +/// any character but those. +pub fn fnmatch(path: &str, pattern: &str) -> bool { + let path: Vec = path.chars().collect(); + let pattern: Vec = pattern.chars().collect(); + matches(&path, &pattern) +} + +// cup: allow-plain shared by the collect Filter and its tests: a glob match, as Python's fnmatch has it +fn matches(path: &[char], pattern: &[char]) -> bool { + match pattern.split_first() { + None => path.is_empty(), + Some(('*', rest)) => (0..=path.len()).any(|skip| matches(&path[skip..], rest)), + Some(('?', rest)) => !path.is_empty() && matches(&path[1..], rest), + Some(('[', rest)) => match (path.split_first(), rest.iter().position(|c| *c == ']')) { + (Some((c, path_rest)), Some(end)) => { + let (negated, class) = match rest[..end].split_first() { + Some(('!', class)) => (true, class), + _ => (false, &rest[..end]), + }; + in_class(*c, class) != negated && matches(path_rest, &rest[end + 1..]) + } + (Some((c, path_rest)), None) => *c == '[' && matches(path_rest, rest), + (None, _) => false, + }, + Some((p, rest)) => path.first() == Some(p) && matches(&path[1..], rest), + } +} + +// cup: allow-plain shared by the collect Filter and its tests: a glob match, as Python's fnmatch has it +fn in_class(c: char, class: &[char]) -> bool { + let mut at = 0; + while at < class.len() { + if at + 2 < class.len() && class[at + 1] == '-' { + if class[at] <= c && c <= class[at + 2] { + return true; + } + at += 3; + } else { + if class[at] == c { + return true; + } + at += 1; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::fnmatch; + + #[test] + fn matches_as_a_shell_glob() { + assert!(fnmatch("src/generated/x.rs", "*/generated/*")); + assert!(fnmatch("a/b.rs", "*.rs")); + assert!(!fnmatch("a/b.rs", "*.ts")); + assert!(fnmatch("x1.rs", "x?.rs")); + assert!(fnmatch("xb.rs", "x[abc].rs")); + assert!(!fnmatch("xd.rs", "x[abc].rs")); + assert!(fnmatch("xd.rs", "x[!abc].rs")); + assert!(fnmatch("x5.rs", "x[0-9].rs")); + assert!(fnmatch("x[.rs", "x[.rs")); + assert!(!fnmatch("discount.rs", "*/discount.rs")); + } +} diff --git a/packages/codeupipe-lint-rs/src/format_report_filter.rs b/packages/codeupipe-lint-rs/src/format_report_filter.rs new file mode 100644 index 0000000..bc7fc2e --- /dev/null +++ b/packages/codeupipe-lint-rs/src/format_report_filter.rs @@ -0,0 +1,62 @@ +use codeupipe::{Filter, StepError}; + +use crate::lint_run::LintRun; + +/// The findings as sorted report lines, and the exit code. +/// +/// Reads `findings`, `suppressed`, `errors` and `strict`. Writes `findings` +/// (sorted), `report` (text for stdout), `error_report` (text for stderr), +/// and `exit_code`: 0 no findings, 1 findings, 2 a path could not be read, +/// so the check is not complete. +/// +/// A warning is in the report and is counted apart. It does not make the +/// exit code 1, unless the run is strict. +pub struct FormatReportFilter; + +impl Filter for FormatReportFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + run.findings.sort_by(|a, b| { + (&a.path, a.line, a.col, &a.rule).cmp(&(&b.path, b.line, b.col, &b.rule)) + }); + let warnings = run + .findings + .iter() + .filter(|finding| finding.warning) + .count(); + let real = run.findings.len() - warnings; + let mut summary = match real { + 0 => "No findings".to_string(), + 1 => "1 finding".to_string(), + _ => format!("{real} findings"), + }; + if !run.suppressed.is_empty() { + summary += &format!(" ({} suppressed)", run.suppressed.len()); + } + summary += "."; + match warnings { + 0 => {} + 1 => summary += " 1 warning.", + _ => summary += &format!(" {warnings} warnings."), + } + let mut lines: Vec = run + .findings + .iter() + .map(|finding| finding.format()) + .collect(); + lines.push(summary); + run.report = lines.join("\n"); + run.error_report = run + .errors + .iter() + .map(|error| format!("error: {error}")) + .collect::>() + .join("\n"); + let fails = real > 0 || (warnings > 0 && run.strict); + run.exit_code = match (run.errors.is_empty(), fails) { + (false, _) => 2, + (true, true) => 1, + (true, false) => 0, + }; + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/lib.rs b/packages/codeupipe-lint-rs/src/lib.rs new file mode 100644 index 0000000..3d052f0 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/lib.rs @@ -0,0 +1,46 @@ +//! A linter for Rust code that uses codeupipe. It reads the source and does +//! not run it. +//! +//! The linter is a codeupipe pipeline ([`LintPipeline`]) that carries a type +//! of its own, [`LintRun`], in place of a `Payload`: a run holds syntax +//! trees, which are not JSON-shaped values. +//! +//! ```no_run +//! use codeupipe_lint::{lint_paths, LintOptions}; +//! +//! let out = lint_paths(&["src".to_string()], &LintOptions::default())?; +//! println!("{}", out.report); +//! # Ok::<(), codeupipe::StepError>(()) +//! ``` + +mod analyze_plain_logic_filter; +mod analyze_primitive_names_filter; +mod analyze_primitives_per_file_filter; +mod analyze_stale_markers_filter; +mod analyze_state_reads_filter; +mod analyze_steps_filter; +mod apply_suppressions_filter; +mod collect_files_filter; +mod drop_disabled_filter; +mod finding; +mod fnmatch; +mod format_report_filter; +mod lint_paths; +mod lint_pipeline; +mod lint_run; +mod logic; +mod markers; +mod parse_files_filter; +mod primitive_kind; +mod read_config; +mod rules; +mod source; +mod step_visitor; + +pub use finding::Finding; +pub use lint_paths::{lint_paths, LintOptions}; +pub use lint_pipeline::LintPipeline; +pub use lint_run::LintRun; +pub use read_config::{read_lint_config, LintConfig, CONFIG_NAME}; +pub use rules::{Rule, RULES}; +pub use source::{Marker, Source}; diff --git a/packages/codeupipe-lint-rs/src/lint_paths.rs b/packages/codeupipe-lint-rs/src/lint_paths.rs new file mode 100644 index 0000000..5ad5515 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/lint_paths.rs @@ -0,0 +1,26 @@ +use codeupipe::{Pipeline, StepError}; + +use crate::lint_pipeline::LintPipeline; +use crate::lint_run::LintRun; + +/// How a run is done: the files that are left out, the rules that are off, +/// and whether a warning fails the run. +#[derive(Clone, Debug, Default)] +pub struct LintOptions { + pub exclude: Vec, + pub disable: Vec, + pub strict: bool, +} + +/// Run the linter over `paths` (files or directories). The run that comes +/// back holds `findings`, `suppressed`, `errors`, `report`, `error_report` +/// and `exit_code`. +pub fn lint_paths(paths: &[String], options: &LintOptions) -> Result { + Pipeline::named(LintPipeline).run(&LintRun { + paths: paths.to_vec(), + exclude: options.exclude.clone(), + disable: options.disable.clone(), + strict: options.strict, + ..LintRun::default() + }) +} diff --git a/packages/codeupipe-lint-rs/src/lint_pipeline.rs b/packages/codeupipe-lint-rs/src/lint_pipeline.rs new file mode 100644 index 0000000..7d8f1fc --- /dev/null +++ b/packages/codeupipe-lint-rs/src/lint_pipeline.rs @@ -0,0 +1,33 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::analyze_plain_logic_filter::AnalyzePlainLogicFilter; +use crate::analyze_primitive_names_filter::AnalyzePrimitiveNamesFilter; +use crate::analyze_primitives_per_file_filter::AnalyzePrimitivesPerFileFilter; +use crate::analyze_stale_markers_filter::AnalyzeStaleMarkersFilter; +use crate::analyze_state_reads_filter::AnalyzeStateReadsFilter; +use crate::analyze_steps_filter::AnalyzeStepsFilter; +use crate::apply_suppressions_filter::ApplySuppressionsFilter; +use crate::collect_files_filter::CollectFilesFilter; +use crate::drop_disabled_filter::DropDisabledFilter; +use crate::format_report_filter::FormatReportFilter; +use crate::lint_run::LintRun; +use crate::parse_files_filter::ParseFilesFilter; + +/// The linter, as a codeupipe pipeline: `Pipeline::named(LintPipeline)`. +pub struct LintPipeline; + +impl NamedPipeline for LintPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(CollectFilesFilter); // paths to files + pipeline.add_filter(ParseFilesFilter); // files to syntax trees (CUP-I000) + pipeline.add_filter(AnalyzeStepsFilter); // CUP-A003, A005 + pipeline.add_filter(AnalyzeStateReadsFilter); // CUP-I003 + pipeline.add_filter(AnalyzePrimitivesPerFileFilter); // CUP-I005 + pipeline.add_filter(AnalyzePrimitiveNamesFilter); // CUP-I006, CUP-I007 + pipeline.add_filter(AnalyzePlainLogicFilter); // CUP-I008, a warning + pipeline.add_filter(ApplySuppressionsFilter); + pipeline.add_filter(AnalyzeStaleMarkersFilter); // CUP-I009, a warning + pipeline.add_filter(DropDisabledFilter); // last, so that a comment for a rule that is off is not stale + pipeline.add_filter(FormatReportFilter); + } +} diff --git a/packages/codeupipe-lint-rs/src/lint_run.rs b/packages/codeupipe-lint-rs/src/lint_run.rs new file mode 100644 index 0000000..0d8adbe --- /dev/null +++ b/packages/codeupipe-lint-rs/src/lint_run.rs @@ -0,0 +1,38 @@ +use codeupipe::PipelineData; + +use crate::finding::Finding; +use crate::source::Source; + +/// What the lint pipeline carries from step to step: the input of a run, +/// and what each step adds. +#[derive(Clone, Default)] +pub struct LintRun { + /// Files or directories to check. + pub paths: Vec, + /// Globs matched against the path of each file; a match is not checked. + pub exclude: Vec, + /// Rule ids that are switched off. + pub disable: Vec, + /// A warning fails the run too. + pub strict: bool, + /// The Rust files under `paths` (CollectFilesFilter). + pub files: Vec, + /// The files that parse (ParseFilesFilter). + pub sources: Vec, + /// The findings that are reported. + pub findings: Vec, + /// The findings that a comment allowed (ApplySuppressionsFilter). + pub suppressed: Vec, + /// Each comment that allowed a finding, as (path, line). + pub used_markers: Vec<(String, usize)>, + /// A path that could not be read: the check is not complete. + pub errors: Vec, + /// The report for stdout (FormatReportFilter). + pub report: String, + /// The errors for stderr. + pub error_report: String, + /// 0 no findings, 1 findings, 2 a path could not be read. + pub exit_code: i32, +} + +impl PipelineData for LintRun {} diff --git a/packages/codeupipe-lint-rs/src/logic.rs b/packages/codeupipe-lint-rs/src/logic.rs new file mode 100644 index 0000000..49483dd --- /dev/null +++ b/packages/codeupipe-lint-rs/src/logic.rs @@ -0,0 +1,48 @@ +use syn::visit::{self, Visit}; + +/// Whether `node` has a decision or a loop: `if`, `match`, `for`, `while`, +/// `loop`, or `let ... else`. A loop that is a method call (`map`, +/// `filter`, `for_each`), `&&`, `||` and `?` are not seen, and nor is code +/// in a macro call. +/// +/// `walk` gives the visitor the node: `has_logic(|v| v.visit_block(block))`. +pub fn has_logic(walk: impl FnOnce(&mut LogicVisitor)) -> bool { + let mut visitor = LogicVisitor { found: false }; + walk(&mut visitor); + visitor.found +} + +pub struct LogicVisitor { + found: bool, +} + +impl<'ast> Visit<'ast> for LogicVisitor { + fn visit_expr_if(&mut self, _: &'ast syn::ExprIf) { + self.found = true; + } + + fn visit_expr_match(&mut self, _: &'ast syn::ExprMatch) { + self.found = true; + } + + fn visit_expr_for_loop(&mut self, _: &'ast syn::ExprForLoop) { + self.found = true; + } + + fn visit_expr_while(&mut self, _: &'ast syn::ExprWhile) { + self.found = true; + } + + fn visit_expr_loop(&mut self, _: &'ast syn::ExprLoop) { + self.found = true; + } + + fn visit_local(&mut self, local: &'ast syn::Local) { + let diverges = local + .init + .as_ref() + .is_some_and(|init| init.diverge.is_some()); + self.found |= diverges; + visit::visit_local(self, local); + } +} diff --git a/packages/codeupipe-lint-rs/src/main.rs b/packages/codeupipe-lint-rs/src/main.rs new file mode 100644 index 0000000..d0bcd69 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/main.rs @@ -0,0 +1,108 @@ +//! codeupipe-lint [--exclude GLOB] [--disable RULE] [--strict] [--config FILE] [--rules] [paths...] +//! +//! With no paths, they come from cup.config.json. Exit code: 0 no findings, +//! 1 findings, 2 a path could not be read or the arguments cannot be used. +//! A warning does not make the exit code 1, unless --strict is given. + +use std::process::ExitCode; + +use codeupipe_lint::{lint_paths, read_lint_config, LintOptions, CONFIG_NAME, RULES}; + +/// The arguments of the command. +#[derive(Default)] +struct Args { + paths: Vec, + exclude: Option>, + disable: Option>, + config: Option, + rules: bool, + strict: bool, +} + +// cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result +fn main() -> ExitCode { + match run(std::env::args().skip(1).collect()) { + Ok(code) => ExitCode::from(code), + Err(error) => { + eprintln!("error: {error}"); + ExitCode::from(2) + } + } +} + +// cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result +fn run(argv: Vec) -> Result { + let mut args = parse(argv)?; + if args.rules { + for rule in RULES { + println!("{} {}", rule.id, rule.slug); + } + return Ok(0); + } + if args.paths.is_empty() { + // The file stands in for the arguments only when no paths are given: + // a path that someone names is checked, whatever the file excludes. + let lint = read_lint_config(args.config.as_deref())?.unwrap_or_default(); + args.paths = lint.paths.unwrap_or_default(); + args.exclude = args.exclude.or(lint.exclude); + args.disable = args.disable.or(lint.disable); + } + if args.paths.is_empty() { + return Err(format!( + "give at least one path, or list them under lint.paths in {CONFIG_NAME}" + )); + } + let disable = args.disable.unwrap_or_default(); + let unknown: Vec<&str> = disable + .iter() + .filter(|id| !RULES.iter().any(|rule| rule.id == id.as_str())) + .map(String::as_str) + .collect(); + if !unknown.is_empty() { + return Err(format!( + "no such rule: {} (the rules are listed by --rules)", + unknown.join(", ") + )); + } + let options = LintOptions { + exclude: args.exclude.unwrap_or_default(), + disable, + strict: args.strict, + }; + let out = lint_paths(&args.paths, &options).map_err(|error| error.to_string())?; + println!("{}", out.report); + if !out.error_report.is_empty() { + eprintln!("{}", out.error_report); + } + Ok(out.exit_code as u8) +} + +// cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result +fn parse(argv: Vec) -> Result { + let mut args = Args::default(); + let mut argv = argv.into_iter(); + while let Some(arg) = argv.next() { + let (name, inline) = match arg.split_once('=') { + Some((name, value)) if name.starts_with("--") => { + (name.to_string(), Some(value.to_string())) + } + _ => (arg.clone(), None), + }; + let mut value = || { + inline + .clone() + .or_else(|| argv.next()) + .ok_or_else(|| format!("option '{name}' needs a value")) + }; + match name.as_str() { + "--exclude" => args.exclude.get_or_insert_with(Vec::new).push(value()?), + "--disable" => args.disable.get_or_insert_with(Vec::new).push(value()?), + "--config" => args.config = Some(value()?), + "--rules" => args.rules = true, + "--strict" => args.strict = true, + _ if name.starts_with("--") => return Err(format!("unknown option '{name}'")), + _ => args.paths.push(arg), + } + } + Ok(args) +} diff --git a/packages/codeupipe-lint-rs/src/markers.rs b/packages/codeupipe-lint-rs/src/markers.rs new file mode 100644 index 0000000..0b80b28 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/markers.rs @@ -0,0 +1,169 @@ +//! The `cup: allow-...` comments of a file. +//! +//! A syntax tree has no comments, so they are found between the tokens: the +//! text between two tokens is space and comments only. A doc comment is a +//! token (an attribute), so text in it is documentation, not a marker, and +//! so is text in a string. + +use proc_macro2::{LineColumn, TokenStream, TokenTree}; + +use crate::source::Marker; + +/// Each line of a comment of `text` that starts with `cup: allow-...`. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +pub fn markers_of(text: &str) -> Vec { + let Ok(tokens) = text.parse::() else { + return Vec::new(); + }; + let lines = Lines::new(text); + let mut taken = Vec::new(); + spans_of(tokens, &mut taken); + let mut taken: Vec<(usize, usize)> = taken + .into_iter() + .map(|(start, end)| (lines.offset(start), lines.offset(end))) + .collect(); + taken.sort_unstable(); + let mut markers = Vec::new(); + let mut at = 0; + for (start, end) in taken { + comments_in(text, &lines, at, start, &mut markers); + at = at.max(end); + } + comments_in(text, &lines, at, text.len(), &mut markers); + markers +} + +/// Where each token starts and ends. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +fn spans_of(tokens: TokenStream, out: &mut Vec<(LineColumn, LineColumn)>) { + for token in tokens { + if let TokenTree::Group(group) = &token { + out.push((group.span_open().start(), group.span_open().end())); + out.push((group.span_close().start(), group.span_close().end())); + spans_of(group.stream(), out); + } else { + out.push((token.span().start(), token.span().end())); + } + } +} + +/// The comments in `text[from..to]`, text that holds space and comments only. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +fn comments_in(text: &str, lines: &Lines, from: usize, to: usize, out: &mut Vec) { + let mut at = from; + while at < to { + let rest = &text[at..to]; + let end = if rest.starts_with("//") { + at + rest.find('\n').unwrap_or(rest.len()) + } else if rest.starts_with("/*") { + at + block_comment_length(rest) + } else { + at += rest.chars().next().map_or(1, char::len_utf8); + continue; + }; + markers_in_comment(text, lines, at, end, out); + at = end; + } +} + +/// The length of the block comment at the start of `text`. Block comments nest. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +fn block_comment_length(text: &str) -> usize { + let mut depth = 0; + let mut at = 0; + while at < text.len() { + let rest = &text[at..]; + if rest.starts_with("/*") { + depth += 1; + at += 2; + } else if rest.starts_with("*/") { + depth -= 1; + at += 2; + if depth == 0 { + return at; + } + } else { + at += rest.chars().next().map_or(1, char::len_utf8); + } + } + text.len() +} + +/// Each line of the comment `text[from..to]` that starts with a marker. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +fn markers_in_comment(text: &str, lines: &Lines, from: usize, to: usize, out: &mut Vec) { + let mut at = from; + for (index, piece) in text[from..to].split('\n').enumerate() { + let line = lines.line_of(at); + let line_start = lines.starts[line - 1]; + let lead = piece.len() - strip_comment_lead(piece).len(); + if let Some(kind) = marker_kind(&piece[lead..]) { + out.push(Marker { + line, + col: text[line_start..at + lead].chars().count() + 1, + kind, + // The first line of the comment may have code before it. + alone: index > 0 || text[line_start..at].trim().is_empty(), + }); + } + at += piece.len() + 1; + } +} + +/// `piece` without the space and the comment characters at its start: `//`, +/// `/*`, or the `*` of a line inside a block comment. +// cup: allow-plain used by the parse Filter: it finds the comments of a file, which a syntax tree does not keep +fn strip_comment_lead(piece: &str) -> &str { + let piece = piece.trim_start(); + let piece = if piece.starts_with("//") { + piece.trim_start_matches('/') + } else if let Some(rest) = piece.strip_prefix("/*") { + rest.trim_start_matches('*') + } else if piece.starts_with('*') && !piece.starts_with("*/") { + &piece[1..] + } else { + piece + }; + piece.trim_start() +} + +/// `allow-plain` for `cup: allow-plain the reason`. +fn marker_kind(text: &str) -> Option { + let rest = text.strip_prefix("cup:")?.trim_start(); + rest.starts_with("allow-").then(|| { + rest.chars() + .take_while(|c| c.is_alphanumeric() || *c == '_' || *c == '-') + .collect() + }) +} + +/// The byte offset where each line of a text starts. +struct Lines<'a> { + text: &'a str, + starts: Vec, +} + +impl<'a> Lines<'a> { + fn new(text: &'a str) -> Self { + let breaks = text.match_indices('\n').map(|(at, _)| at + 1); + Lines { + text, + starts: std::iter::once(0).chain(breaks).collect(), + } + } + + /// The byte offset of a line (from 1) and column (from 0, in characters). + fn offset(&self, at: LineColumn) -> usize { + let start = self.starts[at.line - 1]; + start + + self.text[start..] + .char_indices() + .nth(at.column) + .map_or(self.text.len() - start, |(index, _)| index) + } + + /// The line (from 1) of a byte offset. + fn line_of(&self, offset: usize) -> usize { + self.starts.partition_point(|start| *start <= offset) + } +} diff --git a/packages/codeupipe-lint-rs/src/parse_files_filter.rs b/packages/codeupipe-lint-rs/src/parse_files_filter.rs new file mode 100644 index 0000000..160fcbc --- /dev/null +++ b/packages/codeupipe-lint-rs/src/parse_files_filter.rs @@ -0,0 +1,53 @@ +use std::fs; + +use codeupipe::{Filter, StepError}; + +use crate::finding::Finding; +use crate::lint_run::LintRun; +use crate::markers::markers_of; +use crate::rules::{rule, PARSE_ERROR}; +use crate::source::{position, Source}; + +/// Reads and parses each file. +/// +/// Reads `files`. Writes `sources` (the files that parse, with their +/// `cup: allow-...` comments), a CUP-I000 finding for a file with a syntax +/// error, and `errors` for a file that cannot be read. +pub struct ParseFilesFilter; + +impl Filter for ParseFilesFilter { + fn call(&self, run: &mut LintRun) -> Result<(), StepError> { + for path in run.files.clone() { + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(error) => { + run.errors.push(format!("{path}: {error}")); + continue; + } + }; + match syn::parse_file(&text) { + Ok(tree) => run.sources.push(Source { + markers: markers_of(&text), + path, + text, + tree, + }), + Err(error) => { + let (line, col) = position(error.span()); + let parse_error = rule(PARSE_ERROR).expect("a rule"); + run.findings.push(Finding { + path, + line, + col, + rule: PARSE_ERROR.to_string(), + slug: parse_error.slug.to_string(), + message: format!("{error}. {}", parse_error.fix), + call_line: None, + warning: false, + }); + } + } + } + Ok(()) + } +} diff --git a/packages/codeupipe-lint-rs/src/primitive_kind.rs b/packages/codeupipe-lint-rs/src/primitive_kind.rs new file mode 100644 index 0000000..d3bc3f8 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/primitive_kind.rs @@ -0,0 +1,182 @@ +use std::collections::HashMap; +use std::path::Path; + +use proc_macro2::Span; +use syn::spanned::Spanned; +use syn::{Attribute, Item, Meta, Type}; + +/// The primitives that a type can be. A type is a primitive when it +/// implements the trait: `impl Filter for DiscountFilter`. A named pipeline +/// implements `NamedPipeline`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum PrimitiveKind { + Pipeline, + Hook, + Filter, + Tap, + Predicate, +} + +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +impl PrimitiveKind { + /// The kind of the trait at the end of `path` (`codeupipe::Filter` is a Filter). + pub fn of_trait(path: &syn::Path) -> Option { + let name = path.segments.last()?.ident.to_string(); + [ + ("NamedPipeline", PrimitiveKind::Pipeline), + ("Hook", PrimitiveKind::Hook), + ("Filter", PrimitiveKind::Filter), + ("Tap", PrimitiveKind::Tap), + ("Predicate", PrimitiveKind::Predicate), + ] + .into_iter() + .find(|(trait_name, _)| *trait_name == name) + .map(|(_, kind)| kind) + } + + /// The role word: `Filter`, `Pipeline`, ... + pub fn name(self) -> &'static str { + match self { + PrimitiveKind::Pipeline => "Pipeline", + PrimitiveKind::Hook => "Hook", + PrimitiveKind::Filter => "Filter", + PrimitiveKind::Tap => "Tap", + PrimitiveKind::Predicate => "Predicate", + } + } +} + +/// A type at the top level of a file that is a primitive. +pub struct Primitive { + pub name: String, + pub kind: PrimitiveKind, + /// The name in the declaration of the type, or in the first `impl` of a + /// primitive trait when the type is declared in another file. + pub at: Span, +} + +/// The name of the type that an `impl` is for: `DiscountFilter` for +/// `impl Filter for DiscountFilter` and for `impl

Filter

for DiscountFilter

`. +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +pub fn self_type_name(self_ty: &Type) -> Option { + match self_ty { + Type::Path(path) => path.path.segments.last().map(|s| s.ident.to_string()), + _ => None, + } +} + +/// The primitives among `items`, in the order of the file. A type that +/// implements two primitive traits is one primitive, of the first kind of +/// Pipeline, Hook, Filter, Tap, Predicate. +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +pub fn primitives_in(items: &[Item]) -> Vec { + let mut declared = HashMap::new(); + let mut found: Vec = Vec::new(); + for item in items + .iter() + .filter(|item| !is_test_only(attributes_of(item))) + { + match item { + Item::Struct(item) => { + declared.insert(item.ident.to_string(), item.ident.span()); + } + Item::Enum(item) => { + declared.insert(item.ident.to_string(), item.ident.span()); + } + Item::Impl(item) => { + let kind = item + .trait_ + .as_ref() + .and_then(|(_, path, _)| PrimitiveKind::of_trait(path)); + let (Some(kind), Some(name)) = (kind, self_type_name(&item.self_ty)) else { + continue; + }; + match found.iter_mut().find(|primitive| primitive.name == name) { + Some(primitive) => primitive.kind = primitive.kind.min(kind), + None => found.push(Primitive { + name, + kind, + at: item.self_ty.span(), + }), + } + } + _ => {} + } + } + for primitive in &mut found { + if let Some(at) = declared.get(&primitive.name) { + primitive.at = *at; + } + } + found.sort_by_key(|primitive| { + let start = primitive.at.start(); + (start.line, start.column) + }); + found +} + +/// The names of the types that are primitives, at any level of the file. +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +pub fn primitive_names(items: &[Item], out: &mut Vec) { + for item in items { + match item { + Item::Impl(item) => { + let is_primitive = item + .trait_ + .as_ref() + .and_then(|(_, path, _)| PrimitiveKind::of_trait(path)) + .is_some(); + out.extend(self_type_name(&item.self_ty).filter(|_| is_primitive)); + } + Item::Mod(item) => { + if let Some((_, items)) = &item.content { + primitive_names(items, out); + } + } + _ => {} + } + } +} + +/// A file under a `tests` directory: an integration test of Cargo. +pub fn is_test_file(path: &str) -> bool { + let path = Path::new(path); + path.parent() + .is_some_and(|parent| parent.components().any(|part| part.as_os_str() == "tests")) +} + +/// The attributes of an item. +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +pub fn attributes_of(item: &Item) -> &[Attribute] { + match item { + Item::Const(item) => &item.attrs, + Item::Enum(item) => &item.attrs, + Item::Fn(item) => &item.attrs, + Item::Impl(item) => &item.attrs, + Item::Mod(item) => &item.attrs, + Item::Static(item) => &item.attrs, + Item::Struct(item) => &item.attrs, + Item::Trait(item) => &item.attrs, + Item::Type(item) => &item.attrs, + Item::Union(item) => &item.attrs, + Item::Use(item) => &item.attrs, + _ => &[], + } +} + +/// `#[test]`, or `#[cfg(test)]` and the like: code for the tests only. +// cup: allow-plain shared by the analysis Filters: it says which primitive a type is, from its trait +pub fn is_test_only(attributes: &[Attribute]) -> bool { + attributes.iter().any(|attribute| match &attribute.meta { + Meta::Path(path) => path.is_ident("test"), + Meta::List(list) => { + list.path.is_ident("cfg") + && list + .tokens + .clone() + .into_iter() + .any(|token| token.to_string() == "test") + } + Meta::NameValue(_) => false, + }) +} diff --git a/packages/codeupipe-lint-rs/src/read_config.rs b/packages/codeupipe-lint-rs/src/read_config.rs new file mode 100644 index 0000000..5713618 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/read_config.rs @@ -0,0 +1,67 @@ +use std::fs; +use std::path::Path; + +use serde_json::Value; + +/// The file that holds the settings of the project's tools. +pub const CONFIG_NAME: &str = "cup.config.json"; +const LINT_KEYS: &[&str] = &["paths", "exclude", "disable"]; + +/// The `lint` section of the config file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LintConfig { + pub paths: Option>, + pub exclude: Option>, + pub disable: Option>, +} + +/// The `lint` section of the config file, checked. `None` when the file is +/// not there. With `path` given, a file that is not there is an error. +/// +/// A key that the linter does not read is an error, so the file cannot hold +/// a setting that does nothing. Other sections of the file are for other +/// tools. +// cup: allow-plain the edge of the program: it reads the config file before the pipeline runs +pub fn read_lint_config(path: Option<&str>) -> Result, String> { + let name = path.unwrap_or(CONFIG_NAME); + if !Path::new(name).exists() { + return match path { + None => Ok(None), + Some(_) => Err(format!("{name}: no such file")), + }; + } + let text = fs::read_to_string(name).map_err(|error| format!("{name}: {error}"))?; + let config: Value = serde_json::from_str(&text) + .map_err(|error| format!("{name} is not valid JSON: {error}"))?; + let Value::Object(config) = config else { + return Err(format!("{name} must hold a JSON object")); + }; + let lint = match config.get("lint") { + None => return Ok(Some(LintConfig::default())), + Some(Value::Object(lint)) => lint, + Some(_) => return Err(format!("{name}: lint must be an object")), + }; + let mut out = LintConfig::default(); + for (key, value) in lint { + let strings: Option> = value.as_array().and_then(|items| { + items + .iter() + .map(|item| item.as_str().map(String::from)) + .collect() + }); + let slot = match key.as_str() { + "paths" => &mut out.paths, + "exclude" => &mut out.exclude, + "disable" => &mut out.disable, + _ => { + return Err(format!( + "{name}: lint: unknown key '{key}' (the keys are {})", + LINT_KEYS.join(", ") + )) + } + }; + *slot = + Some(strings.ok_or_else(|| format!("{name}: lint.{key} must be a list of strings"))?); + } + Ok(Some(out)) +} diff --git a/packages/codeupipe-lint-rs/src/rules.rs b/packages/codeupipe-lint-rs/src/rules.rs new file mode 100644 index 0000000..0254ef0 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/rules.rs @@ -0,0 +1,119 @@ +use proc_macro2::Span; + +use crate::finding::Finding; +use crate::source::{position, Source}; + +pub const PARSE_ERROR: &str = "CUP-I000"; +pub const LOCAL_TYPE_STEP: &str = "CUP-A003"; +pub const UNNAMED_PIPELINE: &str = "CUP-A005"; +pub const STATE_READ: &str = "CUP-I003"; +pub const ONE_PRIMITIVE_PER_FILE: &str = "CUP-I005"; +pub const PRIMITIVE_FILE_NAME: &str = "CUP-I006"; +pub const PRIMITIVE_ROLE_SUFFIX: &str = "CUP-I007"; +pub const LOGIC_OUTSIDE_PRIMITIVE: &str = "CUP-I008"; +pub const MARKER_ALLOWS_NOTHING: &str = "CUP-I009"; + +/// One rule: its id, its short name, what to do about a finding, and +/// whether a finding is a warning. +pub struct Rule { + pub id: &'static str, + pub slug: &'static str, + pub fix: &'static str, + pub warning: bool, +} + +const NAMED_TYPE: &str = "Declare it at module level as a named type (a Valve predicate: a type that implements Predicate) \ +so it can be tested on its own."; + +/// The rules, in the order of their ids. The ids and names are those of the +/// Python and TypeScript linters. A rule that the Rust compiler checks +/// (CUP-A001, A002, A008, A009, I004) is not here. +pub const RULES: &[Rule] = &[ + Rule { + id: LOCAL_TYPE_STEP, + slug: "local-type-step", + fix: NAMED_TYPE, + warning: false, + }, + Rule { + id: UNNAMED_PIPELINE, + slug: "unnamed-pipeline-step", + fix: "A nested pipeline must be a named pipeline (a type that implements NamedPipeline, nested as \ +Pipeline::named(UserOnboarding)), named like a filter. A plain Pipeline can only run at the top level.", + warning: false, + }, + Rule { + id: PARSE_ERROR, + slug: "parse-error", + fix: "Fix the syntax error so the file can be checked.", + warning: false, + }, + Rule { + id: STATE_READ, + slug: "state-read-in-step", + fix: "A Filter's result and a Valve's decision must depend only on the Payload. Put what the step needs \ +into the Payload, or observe State from a Tap or a Hook.", + warning: false, + }, + Rule { + id: ONE_PRIMITIVE_PER_FILE, + slug: "one-primitive-per-file", + fix: "Give each Filter, Predicate, Tap, Hook and named Pipeline its own file, so it can be tested alone \ +and used again. The pipeline file is where they come together.", + warning: false, + }, + Rule { + id: PRIMITIVE_FILE_NAME, + slug: "primitive-file-name", + fix: "The file name of a primitive is its type name in snake_case, so one name gives the other.", + warning: false, + }, + Rule { + id: PRIMITIVE_ROLE_SUFFIX, + slug: "primitive-role-suffix", + fix: "A primitive's type name ends with its role (Filter, Tap, Hook, Predicate, Pipeline), so the name \ +tells what the type is.", + warning: false, + }, + Rule { + id: LOGIC_OUTSIDE_PRIMITIVE, + slug: "logic-outside-primitive", + fix: "Write it as a primitive, so that the pipeline has the logic and it can be tested as a step. For \ +logic that must be plain code, say why: // cup: allow-plain ", + warning: true, + }, + Rule { + id: MARKER_ALLOWS_NOTHING, + slug: "marker-allows-nothing", + fix: "Remove it, or put it on the line of the finding that it is for. A comment that allows nothing says \ +that the code has an exception that it does not have.", + warning: true, + }, +]; + +/// The rule with the id `id`. +pub fn rule(id: &str) -> Option<&'static Rule> { + RULES.iter().find(|rule| rule.id == id) +} + +/// A Finding of `rule` at `at`. Its message reads `. `. +pub fn make_finding( + source: &Source, + at: Span, + id: &str, + what: &str, + call_line: Option, +) -> Finding { + let (line, col) = position(at); + let rule = rule(id).expect("a rule of this linter"); + Finding { + path: source.path.clone(), + line, + col, + rule: id.to_string(), + slug: rule.slug.to_string(), + message: format!("{what}. {}", rule.fix), + call_line: call_line.filter(|call_line| *call_line != line), + warning: rule.warning, + } +} diff --git a/packages/codeupipe-lint-rs/src/source.rs b/packages/codeupipe-lint-rs/src/source.rs new file mode 100644 index 0000000..b0cb826 --- /dev/null +++ b/packages/codeupipe-lint-rs/src/source.rs @@ -0,0 +1,53 @@ +use proc_macro2::Span; + +/// One parsed file. +#[derive(Clone)] +pub struct Source { + pub path: String, + pub text: String, + pub tree: syn::File, + /// The `cup: allow-...` comments of the file. + pub markers: Vec, +} + +/// A comment line that starts with `cup: allow-...`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Marker { + pub line: usize, + pub col: usize, + /// `allow-plain`, `allow-name`, ...: the word after `cup:`. + pub kind: String, + /// The line holds the comment and no code. + pub alone: bool, +} + +/// The line and column where `span` starts, from 1. +pub fn position(span: Span) -> (usize, usize) { + let start = span.start(); + (start.line, start.column + 1) +} + +/// The text of `span` in `text`. The lines of a span over several lines are +/// joined with their indent taken away: `self.pipeline.state()`. +// cup: allow-plain shared by the analysis Filters: the text of a node, for a message +pub fn text_of(text: &str, span: Span) -> String { + let (start, end) = (span.start(), span.end()); + text.lines() + .enumerate() + .skip(start.line - 1) + .take(end.line + 1 - start.line) + .map(|(index, line)| { + let to = (index + 1 == end.line).then_some(end.column); + let from = (index + 1 == start.line).then_some(start.column); + let taken: String = line + .chars() + .take(to.unwrap_or(usize::MAX)) + .skip(from.unwrap_or(0)) + .collect(); + match from { + Some(_) => taken, + None => taken.trim_start().to_string(), + } + }) + .collect() +} diff --git a/packages/codeupipe-lint-rs/src/step_visitor.rs b/packages/codeupipe-lint-rs/src/step_visitor.rs new file mode 100644 index 0000000..3869b5b --- /dev/null +++ b/packages/codeupipe-lint-rs/src/step_visitor.rs @@ -0,0 +1,266 @@ +//! Finds the steps that CUP-A003 and CUP-A005 are about, with the names in +//! scope at each `add_...` call. + +use syn::spanned::Spanned; +use syn::visit::{self, Visit}; +use syn::{Expr, Item, Pat, Stmt}; + +use crate::finding::Finding; +use crate::rules::{make_finding, LOCAL_TYPE_STEP, UNNAMED_PIPELINE}; +use crate::source::{position, text_of, Source}; + +/// How far a name is followed, from a binding to the value it was given. +const FOLLOW: usize = 8; + +/// A block, a function or a closure: the names that it declares. +#[derive(Default)] +struct Scope { + /// The types declared in a block. A block's items are seen in all of it. + types: Vec, + /// The bindings, in order: a `let` with its value, or a parameter or a + /// pattern with none. + bindings: Vec<(String, Option)>, + /// A function: it does not see the bindings of the code around it. + function: bool, +} + +/// What is wrong with a step: the rule, and words that say what the value is. +struct Verdict { + rule: &'static str, + what: String, +} + +/// Visits a file and finds the steps of CUP-A003 and CUP-A005. +pub struct StepVisitor<'a> { + source: &'a Source, + scopes: Vec, + pub findings: Vec, +} + +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +impl<'a> StepVisitor<'a> { + pub fn new(source: &'a Source) -> Self { + StepVisitor { + source, + scopes: Vec::new(), + findings: Vec::new(), + } + } + + fn in_scope(&mut self, function: bool, types: Vec, walk: impl FnOnce(&mut Self)) { + self.scopes.push(Scope { + types, + function, + bindings: Vec::new(), + }); + walk(self); + self.scopes.pop(); + } + + fn bind(&mut self, name: String, value: Option) { + if let Some(scope) = self.scopes.last_mut() { + scope.bindings.push((name, value)); + } + } + + /// The value that the binding `name` was given: `None` when the name is + /// not bound, `Some(None)` when it is bound with no value to follow. + fn binding(&self, name: &str) -> Option> { + for scope in self.scopes.iter().rev() { + if let Some((_, value)) = scope.bindings.iter().rev().find(|(bound, _)| bound == name) { + return Some(value.as_ref()); + } + if scope.function { + return None; + } + } + None + } + + /// The type `name` is declared in a block of a function. + fn is_local_type(&self, name: &str) -> bool { + self.scopes + .iter() + .any(|scope| scope.types.iter().any(|declared| declared == name)) + } + + fn check_call(&mut self, call: &syn::ExprMethodCall) { + let roles: &[&str] = match call.method.to_string().as_str() { + "add_filter" => &["Filter"], + "add_tap" => &["Tap"], + "add_pipeline" => &["Pipeline"], + "add_valve" => &["Valve predicate", "Valve filter"], + _ => &[], + }; + let call_line = position(call.method.span()).0; + for (role, argument) in roles.iter().zip(&call.args) { + if let Some(verdict) = self.classify(argument, role, 0) { + let what = format!("{role} {}", verdict.what); + let finding = make_finding( + self.source, + argument.span(), + verdict.rule, + &what, + Some(call_line), + ); + self.findings.push(finding); + } + } + } + + /// What is wrong with `value` as a step of `role`, if anything. + fn classify(&self, value: &Expr, role: &str, depth: usize) -> Option { + if depth > FOLLOW { + return None; + } + match unwrap(value) { + Expr::Path(path) if path.qself.is_none() && path.path.segments.len() == 1 => { + let name = path.path.segments[0].ident.to_string(); + match self.binding(&name) { + Some(Some(bound)) => { + self.classify(bound, role, depth + 1) + .map(|verdict| Verdict { + rule: verdict.rule, + what: format!("`{name}` {}", verdict.what), + }) + } + Some(None) => None, + None => self.local_type(&name), + } + } + Expr::Struct(value) => self.type_of_path(&value.path), + Expr::Call(call) => { + let Expr::Path(function) = &*call.func else { + return None; + }; + let names: Vec = function + .path + .segments + .iter() + .map(|s| s.ident.to_string()) + .collect(); + let names: Vec<&str> = names.iter().map(String::as_str).collect(); + match names.as_slice() { + [.., "Pipeline", "new" | "default"] if role == "Pipeline" => Some(Verdict { + rule: UNNAMED_PIPELINE, + what: format!("is a plain `{}`", text_of(&self.source.text, call.span())), + }), + [.., "Pipeline", "named"] => call + .args + .first() + .and_then(|named| self.classify(named, "named", depth + 1)), + // `LocalTap::new()`, and `LocalTap(1)` for a tuple struct. + [name, _] | [name] => self.local_type(name), + _ => None, + } + } + Expr::MethodCall(call) if call.method == "clone" && call.args.is_empty() => { + self.classify(&call.receiver, role, depth + 1) + } + _ => None, + } + } + + fn type_of_path(&self, path: &syn::Path) -> Option { + match path.segments.len() { + 1 => self.local_type(&path.segments[0].ident.to_string()), + _ => None, + } + } + + fn local_type(&self, name: &str) -> Option { + self.is_local_type(name).then(|| Verdict { + rule: LOCAL_TYPE_STEP, + what: format!("is an instance of `{name}`, a type that is declared in a function"), + }) + } +} + +/// `value` without the parentheses and the `&` around it. +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +fn unwrap(value: &Expr) -> &Expr { + match value { + Expr::Paren(inner) => unwrap(&inner.expr), + Expr::Group(inner) => unwrap(&inner.expr), + Expr::Reference(inner) => unwrap(&inner.expr), + _ => value, + } +} + +/// The name of a pattern that binds one name: `x`, `mut x`, `x: T`. +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +fn name_of(pattern: &Pat) -> Option { + match pattern { + Pat::Ident(ident) if ident.subpat.is_none() => Some(ident.ident.to_string()), + Pat::Type(typed) => name_of(&typed.pat), + _ => None, + } +} + +// cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read source code +impl<'ast> Visit<'ast> for StepVisitor<'_> { + fn visit_block(&mut self, block: &'ast syn::Block) { + let types = block + .stmts + .iter() + .filter_map(|stmt| match stmt { + Stmt::Item(Item::Struct(item)) => Some(item.ident.to_string()), + Stmt::Item(Item::Enum(item)) => Some(item.ident.to_string()), + Stmt::Item(Item::Union(item)) => Some(item.ident.to_string()), + _ => None, + }) + .collect(); + self.in_scope(false, types, |me| visit::visit_block(me, block)); + } + + fn visit_item_fn(&mut self, item: &'ast syn::ItemFn) { + self.in_scope(true, Vec::new(), |me| visit::visit_item_fn(me, item)); + } + + fn visit_impl_item_fn(&mut self, item: &'ast syn::ImplItemFn) { + self.in_scope(true, Vec::new(), |me| visit::visit_impl_item_fn(me, item)); + } + + fn visit_trait_item_fn(&mut self, item: &'ast syn::TraitItemFn) { + self.in_scope(true, Vec::new(), |me| visit::visit_trait_item_fn(me, item)); + } + + fn visit_expr_closure(&mut self, closure: &'ast syn::ExprClosure) { + self.in_scope(false, Vec::new(), |me| { + visit::visit_expr_closure(me, closure) + }); + } + + fn visit_arm(&mut self, arm: &'ast syn::Arm) { + self.in_scope(false, Vec::new(), |me| visit::visit_arm(me, arm)); + } + + fn visit_expr_for_loop(&mut self, for_loop: &'ast syn::ExprForLoop) { + self.in_scope(false, Vec::new(), |me| { + visit::visit_expr_for_loop(me, for_loop) + }); + } + + fn visit_local(&mut self, local: &'ast syn::Local) { + if let Some(init) = &local.init { + self.visit_expr(&init.expr); + if let Some((_, diverge)) = &init.diverge { + self.visit_expr(diverge); + } + } + match name_of(&local.pat) { + Some(name) => self.bind(name, local.init.as_ref().map(|init| (*init.expr).clone())), + None => self.visit_pat(&local.pat), + } + } + + fn visit_pat_ident(&mut self, pattern: &'ast syn::PatIdent) { + self.bind(pattern.ident.to_string(), None); + visit::visit_pat_ident(self, pattern); + } + + fn visit_expr_method_call(&mut self, call: &'ast syn::ExprMethodCall) { + self.check_call(call); + visit::visit_expr_method_call(self, call); + } +} diff --git a/packages/codeupipe-lint-rs/tests/anonymous_steps.rs b/packages/codeupipe-lint-rs/tests/anonymous_steps.rs new file mode 100644 index 0000000..8285ea3 --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/anonymous_steps.rs @@ -0,0 +1,177 @@ +// CUP-A003 and CUP-A005: a step whose type has no place a test can reach, +// and a plain Pipeline nested as a step. A closure or a function as a step +// (CUP-A001, A002, A008) is a compile error in Rust, so it has no rule here. +mod common; + +use common::{disable, lint_in}; + +const HEAD: &str = "use codeupipe::{Filter, NamedPipeline, Payload, Pipeline, Predicate, StepError, Tap}; +struct AddFilter; +impl Filter for AddFilter { fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { Ok(()) } } +struct IsOnPredicate; +impl Predicate for IsOnPredicate { fn test(&self, _payload: &Payload) -> bool { true } } +struct InnerPipeline; +impl NamedPipeline for InnerPipeline { fn add_steps(&self, _pipeline: &mut Pipeline) {} } +"; + +fn build(body: &str) -> common::Linted { + let text = format!("{HEAD}{body}"); + lint_in( + &[("build.rs", &text)], + disable(&["CUP-I005", "CUP-I006", "CUP-I008"]), + ) +} + +#[test] +fn named_types_at_module_level_are_clean() { + let out = build( + "fn build() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_valve(IsOnPredicate, AddFilter); + let kept = AddFilter; + pipeline.add_filter(kept.clone()); + pipeline.add_tap(audit::AuditTap::new()); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); +} +", + ); + assert_eq!(out.found, Vec::::new()); + assert_eq!(out.exit_code, 0); + assert_eq!(out.report, "No findings."); +} + +#[test] +fn a003_a_type_that_is_declared_in_a_function() { + let out = build( + "fn build() { + struct LocalFilter; + let mut pipeline = Pipeline::new(); + pipeline.add_filter(LocalFilter); + let kept = LocalFilter; + pipeline.add_filter(kept); + pipeline.add_valve(IsOnPredicate, LocalFilter {}); + pipeline.add_tap(LocalTap::new()); + pipeline.add_pipeline(Pipeline::named(LocalPipeline)); + struct LocalTap; + struct LocalPipeline; + fn helper(pipeline: &mut Pipeline) { + pipeline.add_filter(LocalFilter); + } +} +", + ); + assert_eq!( + out.found, + [ + "build.rs:11 CUP-A003", + "build.rs:13 CUP-A003", + "build.rs:14 CUP-A003", + "build.rs:15 CUP-A003", + "build.rs:16 CUP-A003", + "build.rs:20 CUP-A003" + ] + ); + assert!(out.findings[0].message.starts_with( + "Filter is an instance of `LocalFilter`, a type that is declared in a function. Declare it" + )); + assert!(out.findings[1] + .message + .starts_with("Filter `kept` is an instance of `LocalFilter`,")); + assert!(out.findings[2].message.starts_with("Valve filter is")); + assert_eq!(out.exit_code, 1); +} + +#[test] +fn a005_a_plain_pipeline_as_a_step() { + let out = build( + "fn build() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::new()); + let mut inner = Pipeline::new(); + inner.add_filter(AddFilter); + pipeline.add_pipeline(inner); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); + pipeline.add_pipeline(Pipeline::::default()); +} +", + ); + assert_eq!( + out.found, + [ + "build.rs:10 CUP-A005", + "build.rs:13 CUP-A005", + "build.rs:15 CUP-A005" + ] + ); + assert!(out.findings[0] + .message + .starts_with("Pipeline is a plain `Pipeline::new()`. A nested pipeline must be")); + assert!(out.findings[1] + .message + .starts_with("Pipeline `inner` is a plain `Pipeline::new()`.")); + assert!(out.findings[2] + .message + .starts_with("Pipeline is a plain `Pipeline::::default()`.")); +} + +#[test] +fn a_name_from_elsewhere_a_parameter_and_a_name_bound_again_are_not_flagged() { + let out = build( + "use other::Imported; +fn build(parameter: Pipeline) { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(Imported); + pipeline.add_pipeline(parameter); + let inner = Pipeline::new(); + let inner = Pipeline::named(InnerPipeline); + pipeline.add_pipeline(inner); + pipeline.add_filter(other::LocalFilter); + let make = |inner: Pipeline| pipeline.add_pipeline(inner); + for inner in list { pipeline.add_pipeline(inner); } +} +", + ); + assert_eq!(out.found, Vec::::new()); +} + +#[test] +fn the_add_steps_of_a_named_pipeline_is_checked_too() { + let out = lint_in( + &[( + "pricing_pipeline.rs", + "use codeupipe::{NamedPipeline, Pipeline}; +pub struct PricingPipeline; +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_pipeline(Pipeline::new()); + } +} +", + )], + Default::default(), + ); + assert_eq!(out.found, ["pricing_pipeline.rs:5 CUP-A005"]); +} + +#[test] +fn allow_anonymous_suppresses_a_finding_on_its_line_the_line_above_or_the_line_of_the_call() { + let out = build( + "fn build() { + struct LocalFilter; + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::new()); // cup: allow-anonymous a test of the refusal + // cup: allow-anonymous + pipeline.add_pipeline(Pipeline::new()); + pipeline.add_valve( // cup: allow-anonymous + IsOnPredicate, + LocalFilter, + ); + pipeline.add_pipeline(Pipeline::new()); +} +", + ); + assert_eq!(out.found, ["build.rs:18 CUP-A005"]); + assert_eq!(out.suppressed, 3); + assert!(out.report.ends_with("1 finding (3 suppressed).")); +} diff --git a/packages/codeupipe-lint-rs/tests/cli.rs b/packages/codeupipe-lint-rs/tests/cli.rs new file mode 100644 index 0000000..805792a --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/cli.rs @@ -0,0 +1,169 @@ +// The command: its exit codes, arguments, config file, and the linter on +// the Rust code of this repository. +mod common; + +use std::path::Path; +use std::process::Command; + +use codeupipe_lint::LintOptions; +use common::{lint_in, project}; + +const BAD: &str = "pub struct Discount; +impl Filter for Discount { fn call(&self, _p: &mut Payload) -> Result<(), StepError> { Ok(()) } } +"; +const GOOD: &str = "pub struct SetFilter; +impl Filter for SetFilter { fn call(&self, _p: &mut Payload) -> Result<(), StepError> { Ok(()) } } +"; + +struct Ran { + code: i32, + out: String, + err: String, +} + +fn run(cwd: &Path, args: &[&str]) -> Ran { + let done = Command::new(env!("CARGO_BIN_EXE_codeupipe-lint")) + .args(args) + .current_dir(cwd) + .output() + .unwrap(); + Ran { + code: done.status.code().unwrap(), + out: String::from_utf8(done.stdout).unwrap(), + err: String::from_utf8(done.stderr).unwrap(), + } +} + +#[test] +fn exit_0_with_no_findings_exit_1_with_findings_one_line_for_each() { + let root = project(&[("discount.rs", BAD), ("set_filter.rs", GOOD)]); + let clean = run(&root, &["set_filter.rs"]); + assert_eq!(clean.code, 0); + assert_eq!(clean.out.trim(), "No findings."); + let bad = run(&root, &["."]); + assert_eq!(bad.code, 1); + assert!( + bad.out + .starts_with("discount.rs:1:12 CUP-I006 primitive-file-name "), + "{}", + bad.out + ); + assert!(bad.out.ends_with("\n2 findings.\n")); +} + +#[test] +fn exit_2_for_a_path_that_is_not_there_a_rule_that_is_not_known_and_no_paths() { + let root = project(&[]); + let missing = run(&root, &["nothing-here"]); + assert_eq!(missing.code, 2); + assert!(missing + .err + .contains("error: nothing-here: no such file or directory")); + let unknown = run(&root, &["--disable", "CUP-X999", "."]); + assert_eq!(unknown.code, 2); + assert!(unknown.err.contains("no such rule: CUP-X999")); + // A rule that the compiler checks in Rust is not a rule of this linter. + assert_eq!(run(&root, &["--disable", "CUP-A001", "."]).code, 2); + assert_eq!(run(&root, &[]).code, 2); + assert_eq!(run(&root, &["--nope", "."]).code, 2); +} + +#[test] +fn rules_lists_the_rule_ids() { + let listed = run(&project(&[]), &["--rules"]); + assert_eq!(listed.code, 0); + assert_eq!( + listed.out, + "CUP-A003 local-type-step +CUP-A005 unnamed-pipeline-step +CUP-I000 parse-error +CUP-I003 state-read-in-step +CUP-I005 one-primitive-per-file +CUP-I006 primitive-file-name +CUP-I007 primitive-role-suffix +CUP-I008 logic-outside-primitive +CUP-I009 marker-allows-nothing +" + ); +} + +#[test] +fn exclude_and_disable() { + let root = project(&[("discount.rs", BAD)]); + assert_eq!( + run( + &root, + &[ + "--exclude", + "*/discount.rs", + "--exclude", + "discount.rs", + "." + ] + ) + .code, + 0 + ); + assert_eq!( + run(&root, &["--disable", "CUP-I006", "--disable=CUP-I007", "."]).code, + 0 + ); + assert_eq!(run(&root, &["--disable", "CUP-I006", "."]).code, 1); +} + +#[test] +fn with_no_paths_they_come_from_cup_config_json() { + let root = project(&[ + ("discount.rs", BAD), + ( + "cup.config.json", + r#"{"lint": {"paths": ["."], "disable": ["CUP-I006", "CUP-I007"]}, "deploy": {}}"#, + ), + ]); + assert_eq!(run(&root, &[]).code, 0); + assert_eq!(run(&root, &["discount.rs"]).code, 1); + std::fs::write(root.join("cup.config.json"), r#"{"lint": {"path": ["."]}}"#).unwrap(); + let wrong = run(&root, &[]); + assert_eq!(wrong.code, 2); + assert!(wrong.err.contains("unknown key 'path'"), "{}", wrong.err); + std::fs::write(root.join("cup.config.json"), r#"{"lint": {"paths": "."}}"#).unwrap(); + assert!(run(&root, &[]) + .err + .contains("lint.paths must be a list of strings")); + assert!(run(&root, &["--config", "other.json"]) + .err + .contains("other.json: no such file")); +} + +#[test] +fn target_and_git_are_not_read() { + let out = lint_in( + &[("target/debug/discount.rs", BAD), (".git/discount.rs", BAD)], + LintOptions::default(), + ); + assert_eq!(out.found, Vec::::new()); +} + +#[test] +fn the_linter_is_clean_on_the_rust_code_of_this_repository() { + let here = Path::new(env!("CARGO_MANIFEST_DIR")); + let done = run( + here, + &[ + "--strict", + "src", + "tests", + "../codeupipe-rs/src", + "../codeupipe-rs/examples", + "../codeupipe-rs/tests", + "../../conformance/runners/rust/src", + ], + ); + // Each plain logic of this repository says why, with `cup: allow-plain`. + assert!( + done.out.starts_with("No findings (") && !done.out.contains("warning"), + "{}", + done.out + ); + assert_eq!(done.code, 0); +} diff --git a/packages/codeupipe-lint-rs/tests/common/mod.rs b/packages/codeupipe-lint-rs/tests/common/mod.rs new file mode 100644 index 0000000..abcb339 --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/common/mod.rs @@ -0,0 +1,69 @@ +// Helpers for the tests: write source text to a new directory and lint it. +#![allow(dead_code)] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use codeupipe_lint::{lint_paths, Finding, LintOptions}; + +static MADE: AtomicUsize = AtomicUsize::new(0); + +/// A new, empty directory with `files` in it (name, text). +pub fn project(files: &[(&str, &str)]) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "cup-lint-rs-{}-{}", + std::process::id(), + MADE.fetch_add(1, Ordering::SeqCst) + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + for (name, text) in files { + let path = root.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + root +} + +pub struct Linted { + /// One string for each finding: `file:line rule`. + pub found: Vec, + pub findings: Vec, + pub suppressed: usize, + pub report: String, + pub exit_code: i32, +} + +pub fn lint_in(files: &[(&str, &str)], options: LintOptions) -> Linted { + let root = project(files); + let out = lint_paths(&[root.to_string_lossy().into_owned()], &options).unwrap(); + let found = out + .findings + .iter() + .map(|f| format!("{}:{} {}", relative(&root, &f.path), f.line, f.rule)) + .collect(); + let _ = fs::remove_dir_all(&root); + Linted { + found, + findings: out.findings, + suppressed: out.suppressed.len(), + report: out.report, + exit_code: out.exit_code, + } +} + +pub fn disable(rules: &[&str]) -> LintOptions { + LintOptions { + disable: rules.iter().map(|rule| rule.to_string()).collect(), + ..LintOptions::default() + } +} + +fn relative(root: &Path, path: &str) -> String { + Path::new(path) + .strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned() +} diff --git a/packages/codeupipe-lint-rs/tests/invariants.rs b/packages/codeupipe-lint-rs/tests/invariants.rs new file mode 100644 index 0000000..7110ddb --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/invariants.rs @@ -0,0 +1,206 @@ +// CUP-I003, I005, I006, I007, and CUP-I000. A write to the Payload outside a +// Filter (CUP-I004) is a compile error in Rust, so it has no rule here. +mod common; + +use codeupipe_lint::LintOptions; +use common::lint_in; + +#[test] +fn i003_a_filter_or_a_predicate_that_reads_state() { + let out = lint_in( + &[ + ( + "count_filter.rs", + "use std::rc::Rc; +use codeupipe::{Filter, Payload, Pipeline, State, StepError}; +pub struct CountFilter { + pipeline: Rc, + seen: Rc, + states: Vec, +} +impl Filter for CountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert(\"a\", self.pipeline.state().executed().len()); + payload.insert(\"b\", self.seen.skipped().len()); + payload.insert(\"c\", self.states.len()); + payload.insert(\"d\", payload.get(\"state\").is_some()); + let state = State::new(); + payload.insert(\"e\", state.executed().len() + self.state()); + Ok(()) + } +} +", + ), + ( + "ran_predicate.rs", + "pub struct RanPredicate { holder: Holder } +impl Predicate for RanPredicate { + fn test(&self, _payload: &Payload) -> bool { !self.holder.state().executed().is_empty() } +} +", + ), + ( + "watch_tap.rs", + "pub struct WatchTap { pipeline: Rc } +impl Tap for WatchTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + println!(\"{:?}\", self.pipeline.state().executed()); + Ok(()) + } +} +", + ), + ], + LintOptions::default(), + ); + assert_eq!( + out.found, + [ + "count_filter.rs:10 CUP-I003", + "count_filter.rs:11 CUP-I003", + "ran_predicate.rs:3 CUP-I003" + ] + ); + assert!(out.findings[0] + .message + .starts_with("Step reads pipeline State (`self.pipeline.state()`). A Filter's result")); + assert!(out.findings[1] + .message + .starts_with("Step reads pipeline State (`self.seen`).")); +} + +const FILTER: &str = "fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { Ok(()) }"; +const TAP: &str = "fn observe(&self, _payload: &Payload) -> Result<(), StepError> { Ok(()) }"; + +#[test] +fn i005_one_primitive_in_each_file() { + let set_filter = format!( + "pub struct SetFilter; +impl Filter for SetFilter {{ {FILTER} }} +struct Helper; +pub struct SeeTap; +impl Tap for SeeTap {{ {TAP} }} +fn make() {{ struct LocalFilter; impl Filter for LocalFilter {{ {FILTER} }} }} +" + ); + let two = format!( + "struct AFilter;\nimpl Filter for AFilter {{ {FILTER} }}\nstruct BFilter;\nimpl Filter for BFilter {{ {FILTER} }}\n" + ); + let out = lint_in( + &[("set_filter.rs", &set_filter), ("tests/steps.rs", &two)], + LintOptions::default(), + ); + assert_eq!( + out.found, + ["set_filter.rs:4 CUP-I005", "set_filter.rs:4 CUP-I006"] + ); + assert!(out.findings[0] + .message + .starts_with("Tap `SeeTap` shares this file with Filter `SetFilter`.")); +} + +#[test] +fn i006_the_file_name_is_the_type_name_in_snake_case() { + let out = lint_in( + &[ + ( + "discount.rs", + &format!("pub struct DiscountFilter;\nimpl Filter for DiscountFilter {{ {FILTER} }}\n"), + ), + ( + "parse_html_filter.rs", + &format!("pub struct ParseHTMLFilter;\nimpl Filter for ParseHTMLFilter {{ {FILTER} }}\n"), + ), + ( + "pricing_pipeline.rs", + "pub struct PricingPipeline;\nimpl NamedPipeline for PricingPipeline { fn add_steps(&self, _pipeline: &mut Pipeline) {} }\n", + ), + ( + "order_filter.rs", + &format!("impl codeupipe::Filter

for OrderFilter

{{ {FILTER} }}\n"), + ), + ], + LintOptions::default(), + ); + assert_eq!(out.found, ["discount.rs:1 CUP-I006"]); + assert!(out.findings[0] + .message + .starts_with("`DiscountFilter` is in `discount.rs`. Put it in `discount_filter.rs`.")); +} + +#[test] +fn i007_the_type_name_ends_with_its_role() { + let out = lint_in( + &[ + ( + "discount.rs", + &format!("pub struct Discount;\nimpl Filter for Discount {{ {FILTER} }}\n"), + ), + ( + "audit.rs", + &format!("pub struct Audit;\nimpl codeupipe::Tap for Audit {{ {TAP} }}\n"), + ), + ( + "at_least.rs", + "pub struct AtLeast;\nimpl Predicate for AtLeast { fn test(&self, _p: &Payload) -> bool { true } }\n", + ), + ( + "print_steps.rs", + "pub struct PrintSteps;\nimpl Hook for PrintSteps {}\n", + ), + ( + "pricing.rs", + "pub struct Pricing;\nimpl NamedPipeline for Pricing { fn add_steps(&self, _pipeline: &mut Pipeline) {} }\n", + ), + ( + "point.rs", + "pub struct Point;\nimpl Point { fn call(&self, a: i32, b: i32) {} fn test(&self) {} }\n", + ), + ], + LintOptions::default(), + ); + assert_eq!( + out.found, + [ + "at_least.rs:1 CUP-I006", + "at_least.rs:1 CUP-I007", + "audit.rs:1 CUP-I006", + "audit.rs:1 CUP-I007", + "discount.rs:1 CUP-I006", + "discount.rs:1 CUP-I007", + "pricing.rs:1 CUP-I006", + "pricing.rs:1 CUP-I007", + "print_steps.rs:1 CUP-I006", + "print_steps.rs:1 CUP-I007" + ] + ); + assert!(out.findings[1] + .message + .starts_with("Predicate `AtLeast` does not say what it is. Name it `AtLeastPredicate`.")); +} + +#[test] +fn the_comment_markers_suppress_the_invariant_findings() { + let out = lint_in( + &[( + "discount.rs", + &format!( + "// cup: allow-name\n#[derive(Clone)]\n// cup: allow-name a name from the paper\npub struct Discount;\nimpl Filter for Discount {{ {FILTER} }}\n" + ), + )], + LintOptions::default(), + ); + // The first marker is not directly above the struct, so it allows nothing. + assert_eq!(out.found, ["discount.rs:1 CUP-I009"]); + assert_eq!(out.suppressed, 2); +} + +#[test] +fn a_file_with_a_syntax_error_is_i000() { + let out = lint_in(&[("broken.rs", "pub struct {{{\n")], LintOptions::default()); + assert_eq!(out.found, ["broken.rs:1 CUP-I000"]); + assert!(out.findings[0] + .message + .ends_with("Fix the syntax error so the file can be checked.")); + assert_eq!(out.exit_code, 1); +} diff --git a/packages/codeupipe-lint-rs/tests/plain_logic.rs b/packages/codeupipe-lint-rs/tests/plain_logic.rs new file mode 100644 index 0000000..68cb9af --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/plain_logic.rs @@ -0,0 +1,239 @@ +// CUP-I008, a warning: logic that is not in a primitive. +mod common; + +use codeupipe_lint::LintOptions; +use common::{disable, lint_in, Linted}; + +fn warnings(out: &Linted) -> Vec { + out.findings + .iter() + .filter(|f| f.warning) + .map(|f| f.message.split(" has logic").next().unwrap().to_string()) + .collect() +} + +#[test] +fn a_function_with_a_decision_that_is_not_in_a_primitive_is_a_warning() { + let out = lint_in( + &[( + "who_is_due.rs", + "pub fn who_is_due(entries: &[String]) -> &'static str { if entries.is_empty() { return \"say\"; } \"model\" } +pub fn pick(list: &[i32]) -> Option { for each in list { if *each > 0 { return Some(*each); } } None } +pub fn choose(flag: bool) -> &'static str { match flag { true => \"a\", false => \"b\" } } +pub fn wait() { loop { break; } } +pub fn plain(a: i32, b: i32) -> i32 { a + b } +pub fn double(list: &[i32]) -> Vec { list.iter().map(|n| n * 2).collect() } +pub fn first(list: &[i32]) -> i32 { let Some(x) = list.first() else { return 0 }; *x } +", + )], + LintOptions::default(), + ); + assert_eq!( + out.found, + [ + "who_is_due.rs:1 CUP-I008", + "who_is_due.rs:2 CUP-I008", + "who_is_due.rs:3 CUP-I008", + "who_is_due.rs:4 CUP-I008", + "who_is_due.rs:7 CUP-I008" + ] + ); + assert_eq!( + warnings(&out), + [ + "Function `who_is_due`", + "Function `pick`", + "Function `choose`", + "Function `wait`", + "Function `first`" + ] + ); + assert!(out.findings[0].message.contains( + "is not a Filter, a Predicate, a Tap, a Hook or a Pipeline. Write it as a primitive" + )); +} + +#[test] +fn a_type_or_a_trait_that_is_not_a_primitive_and_code_at_the_top_of_a_file() { + let out = lint_in( + &[ + ( + "record_store.rs", + "pub struct RecordStore { files: Vec } +impl RecordStore { + pub fn read(&self, id: &str) -> Vec { if id.is_empty() { return vec![]; } self.files.clone() } +} +pub struct Point { x: f64 } +impl Point { pub fn double(&self) -> f64 { self.x * 2.0 } } +impl std::fmt::Display for Point { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, \"{}\", self.x) } } +pub trait Choose { fn choose(&self, a: bool) -> i32 { if a { 1 } else { 2 } } } +impl Default for Elsewhere { fn default() -> Self { match 1 { _ => Elsewhere } } } +", + ), + ( + "main.rs", + "const LIMIT: usize = if cfg!(debug_assertions) { 1 } else { 2 }; +fn main() { let args: Vec = std::env::args().collect(); if args.is_empty() { std::process::exit(2); } } +", + ), + ( + "helpers.rs", + "pub mod inner { + pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } } +} +", + ), + ], + LintOptions::default(), + ); + assert_eq!( + out.found, + [ + "helpers.rs:2 CUP-I008", + "main.rs:1 CUP-I008", + "main.rs:2 CUP-I008", + "record_store.rs:2 CUP-I008", + "record_store.rs:8 CUP-I008", + "record_store.rs:9 CUP-I008" + ] + ); + assert_eq!( + warnings(&out), + [ + "Function `pick`", + "The code at the top of this file", + "Function `main`", + "Type `RecordStore`", + "Trait `Choose`", + "Type `Elsewhere`" + ] + ); +} + +#[test] +fn logic_in_a_primitive_is_where_it_must_be_and_types_and_tests_have_no_finding() { + let out = lint_in( + &[ + ( + "pick_filter.rs", + "pub struct PickFilter { on: bool } +impl PickFilter { pub fn new(on: bool) -> Self { if on { PickFilter { on } } else { PickFilter { on: false } } } } +impl Filter for PickFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.has(\"a\") { payload.insert(\"b\", 1); } + for _ in 0..2 {} + Ok(()) + } +} +", + ), + ( + "is_on_predicate.rs", + "pub struct IsOnPredicate;\nimpl Predicate for IsOnPredicate { fn test(&self, p: &Payload) -> bool { match p.get(\"on\") { Some(_) => true, None => false } } }\n", + ), + ( + "see_tap.rs", + "pub struct SeeTap;\nimpl Tap for SeeTap { fn observe(&self, p: &Payload) -> Result<(), StepError> { if p.has(\"x\") { println!(\"x\"); } Ok(()) } }\n", + ), + ( + "log_hook.rs", + "pub struct LogHook;\nimpl Hook for LogHook { fn before(&self, step: Option<&str>, _p: &Payload) { if let Some(step) = step { println!(\"{step}\"); } } }\n", + ), + ( + "main_pipeline.rs", + "pub struct MainPipeline;\nimpl NamedPipeline for MainPipeline { fn add_steps(&self, pipeline: &mut Pipeline) { if std::env::var(\"X\").is_ok() { pipeline.observe(true, false); } } }\n", + ), + ( + "types.rs", + "pub enum Due { Say, Model } +pub trait Model { fn reply(&self, text: &str) -> String; } +pub const LIMIT: usize = 20000; +pub type Names = Vec; +#[test] +fn checks() { if LIMIT > 1 { return; } } +#[cfg(test)] +mod tests { + fn helper(a: bool) -> i32 { if a { 1 } else { 2 } } +} +", + ), + ], + LintOptions::default(), + ); + assert_eq!(out.found, Vec::::new()); +} + +#[test] +fn a_warning_does_not_fail_the_run_unless_the_run_is_strict() { + let files = [( + "helper.rs", + "pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } }\n", + )]; + let soft = lint_in(&files, LintOptions::default()); + assert_eq!(soft.exit_code, 0); + assert!(soft.report.contains( + "helper.rs:1:8 CUP-I008 logic-outside-primitive warning: Function `pick` has logic" + )); + assert!(soft.report.ends_with("\nNo findings. 1 warning.")); + let strict = lint_in( + &files, + LintOptions { + strict: true, + ..LintOptions::default() + }, + ); + assert_eq!(strict.exit_code, 1); +} + +#[test] +fn a_warning_and_a_finding_are_counted_apart() { + let out = lint_in( + &[( + "discount.rs", + "pub struct Discount; +impl Filter for Discount { fn call(&self, _p: &mut Payload) -> Result<(), StepError> { Ok(()) } } +pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } } +", + )], + LintOptions::default(), + ); + assert_eq!(out.exit_code, 1); + assert!(out.report.ends_with("\n2 findings. 1 warning.")); +} + +#[test] +fn allow_plain_with_a_reason_suppresses_the_warning_and_test_files_are_not_checked() { + let out = lint_in( + &[ + ( + "helper.rs", + "/// Picks one. +#[inline] +// cup: allow-plain a pure helper of two Predicates +pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } } +", + ), + ( + "tests/support.rs", + "pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } }\n", + ), + ], + LintOptions::default(), + ); + assert_eq!(out.found, Vec::::new()); + assert_eq!(out.suppressed, 1); + assert_eq!(out.report, "No findings (1 suppressed)."); +} + +#[test] +fn the_rule_can_be_switched_off() { + let out = lint_in( + &[( + "helper.rs", + "pub fn pick(a: bool) -> i32 { if a { 1 } else { 2 } }\n", + )], + disable(&["CUP-I008"]), + ); + assert_eq!(out.found, Vec::::new()); + assert_eq!(out.report, "No findings."); +} diff --git a/packages/codeupipe-lint-rs/tests/stale_markers.rs b/packages/codeupipe-lint-rs/tests/stale_markers.rs new file mode 100644 index 0000000..8751339 --- /dev/null +++ b/packages/codeupipe-lint-rs/tests/stale_markers.rs @@ -0,0 +1,91 @@ +// CUP-I009, a warning: a `cup: allow-...` comment that allowed no finding. +mod common; + +use codeupipe_lint::LintOptions; +use common::{disable, lint_in}; + +#[test] +fn a_comment_that_allows_nothing_is_a_warning() { + let out = lint_in( + &[( + "add.rs", + "// cup: allow-plain it had a loop before +pub fn add(a: i32, b: i32) -> i32 { a + b } +pub const ONE: i32 = 1; // cup: allow-state-read nothing here reads State +/* + * cup: allow-plian a kind that does not exist + */ +pub const TWO: i32 = 2; +// cup: allow-outside-write Rust has no such rule +pub const THREE: i32 = 3; +", + )], + LintOptions::default(), + ); + assert_eq!( + out.found, + [ + "add.rs:1 CUP-I009", + "add.rs:3 CUP-I009", + "add.rs:5 CUP-I009", + "add.rs:8 CUP-I009" + ] + ); + assert!(out.findings[0].warning); + assert!(out.findings[0] + .message + .starts_with("The comment `cup: allow-plain` allows nothing. Remove it")); + assert_eq!(out.findings[1].col, 28); + assert_eq!(out.exit_code, 0); + let strict = lint_in( + &[( + "add.rs", + "// cup: allow-plain no reason\npub const ONE: i32 = 1;\n", + )], + LintOptions { + strict: true, + ..LintOptions::default() + }, + ); + assert_eq!(strict.exit_code, 1); +} + +#[test] +fn a_comment_that_allows_a_finding_is_not_reported_and_text_that_only_names_a_marker_is_not_a_comment( +) { + let out = lint_in( + &[( + "pick.rs", + "// cup: allow-plain the entry of the program +pub fn pick(list: &[i32]) -> Option { for each in list { if *each > 0 { return Some(*each); } } None } +pub fn choose(flag: bool) -> i32 { if flag { 1 } else { 2 } } // cup: allow-plain a small helper +/// cup: allow-plain in a doc comment is text of the documentation +pub const HELP: &str = \"say why: // cup: allow-plain \"; +pub const RAW: &str = r#\"// cup: allow-plain\"#; +pub const SLASH: char = '/'; // a char, then cup: allow-plain is not at the start +", + )], + LintOptions::default(), + ); + assert_eq!(out.found, Vec::::new()); + assert_eq!(out.suppressed, 2); +} + +#[test] +fn a_comment_for_a_rule_that_is_switched_off_is_not_reported_and_the_rule_can_be_switched_off() { + let files = [( + "pick.rs", + "// cup: allow-plain the entry\npub fn pick(flag: bool) -> i32 { if flag { 1 } else { 2 } }\n", + )]; + let off = lint_in(&files, disable(&["CUP-I008"])); + assert_eq!(off.found, Vec::::new()); + assert_eq!(off.suppressed, 0); + let quiet = lint_in( + &[( + "one.rs", + "// cup: allow-plain no reason\npub const ONE: i32 = 1;\n", + )], + disable(&["CUP-I009"]), + ); + assert_eq!(quiet.found, Vec::::new()); +} diff --git a/packages/codeupipe-lint-ts/.gitignore b/packages/codeupipe-lint-ts/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/packages/codeupipe-lint-ts/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/packages/codeupipe-lint-ts/LICENSE b/packages/codeupipe-lint-ts/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-lint-ts/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-lint-ts/NOTICE b/packages/codeupipe-lint-ts/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-lint-ts/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-lint-ts/README.md b/packages/codeupipe-lint-ts/README.md new file mode 100644 index 0000000..15d02d8 --- /dev/null +++ b/packages/codeupipe-lint-ts/README.md @@ -0,0 +1,143 @@ +# codeupipe-lint for TypeScript + +A linter for TypeScript code that uses codeupipe. It reads the source and +does not run it. + +The linter is a codeupipe pipeline: [src/lint-pipeline.ts](src/lint-pipeline.ts). +It depends on `codeupipe` and on the `typescript` package, which it uses to +parse the files. + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. It is not on npm yet: the +[README of the core package](../codeupipe-ts/README.md) says how to install it +from this repository. + +```bash +npx codeupipe-lint src # files or directories +npx codeupipe-lint --exclude '*/generated/*' src +npx codeupipe-lint --disable CUP-I005 src +npx codeupipe-lint --strict src # a warning fails the run too +npx codeupipe-lint --rules # list the rule ids +npx codeupipe-lint # paths come from cup.config.json +``` + +Each finding is one line, and the message names the fix: + +``` +path:line:col CUP-A001 lambda-step Filter is a function. Define it at module level as a named class (...) +``` + +Exit code: **0** no findings, **1** findings, **2** a path could not be +read or the arguments cannot be used. + +From TypeScript: `lintPaths(paths, { exclude, disable })` returns the result +Payload (`findings`, `suppressed`, `errors`, `report`, `exitCode`). + +## Rules + +The ids are those of the Python linter, where it has the rule. + +| id | slug | what | +|---|---|---| +| CUP-I000 | parse-error | the file has a syntax error | +| CUP-A001 | lambda-step | an arrow function or a function expression as a step | +| CUP-A002 | local-function-step | a named function as a Filter or a Tap | +| CUP-A003 | local-type-step | an instance of a class that is declared in a function, or of a class with no name | +| CUP-A005 | unnamed-pipeline-step | a plain `new Pipeline()` nested as a step | +| CUP-A008 | function-predicate | a named function as a Valve predicate | +| CUP-A009 | object-literal-step | an object literal as a step. TypeScript only | +| CUP-I003 | state-read-in-step | a Filter (`call`) or a Predicate (`test`) reads the State of a pipeline | +| CUP-I004 | write-outside-filter | a Tap, a Predicate or a Hook writes to the Payload | +| CUP-I005 | one-primitive-per-file | a second primitive class at the top level of a file | +| CUP-I006 | primitive-file-name | the file name is not the class name in kebab-case | +| CUP-I007 | primitive-role-suffix | the class name does not end with its role | +| CUP-I008 | logic-outside-primitive | **a warning.** A function, a class or the code at the top of a file has logic (`if`, `? :`, `switch`, a loop, `try`) and is not a primitive | +| CUP-I009 | marker-allows-nothing | **a warning.** A `cup: allow-...` comment that allowed no finding: the code that it was for is gone or has moved, or its kind has a spelling error | + +A step is the argument of `addFilter`, `addTap` or `addPipeline` (on any +receiver), and the two arguments of `addValve(predicate, filter)`. + +A class is a primitive by its shape: `extends Pipeline`, `extends Hook`, +`implements Filter`, `implements Tap`, `implements Predicate`, or a method +`call`, `observe` or `test` with exactly one required parameter. + +## Logic that is not in a primitive (CUP-I008) + +A project that uses codeupipe writes its logic as Filters, Predicates, Taps, +Hooks and Pipelines. Logic in a plain function or a plain class works as +well, so nothing else shows the mistake. This rule shows it. + +``` +path:line:col CUP-I008 logic-outside-primitive warning: Function `whoIsDue` has logic and is not a Filter, ... +``` + +- **It is a warning.** It is in the report and it is counted apart + (`No findings. 3 warnings.`). It does not make the exit code 1. With + `--strict`, a warning fails the run as a finding does. +- **Flagged, once each:** a function at the top of a file, a class at the + top of a file that is not a primitive, and the code at the top of a file. +- **Not flagged:** a primitive, a function or a class with no logic, a type, + and a test file. +- **Not seen:** a loop that is a method call (`map`, `filter`, `forEach`), + and `&&` / `||`. +- For logic that must be plain code (the edge to a model server, the entry + of a program), say why in a comment: `// cup: allow-plain `. + +The Python linter has this rule too. + +## Rules of the Python linter that are not here + +- **CUP-I001** (a change to the Payload behind its API). The data of a + Payload is in a private field (`#data`), so the language refuses this. +- **CUP-I002** (a change to a value that Payloads share). Not built yet. + +## Turn a finding off + +- **One line.** A comment on the line of the finding, on the line of the + `add...` call around it, or alone on the line directly above: + `// cup: allow-anonymous` for a CUP-A finding; + `// cup: allow-state-read`, `allow-outside-write`, `allow-shared-file`, + `allow-name` or `allow-plain` for a CUP-I finding. Any text can follow the marker. + A comment that allows no finding is itself a finding (CUP-I009), so an old one does not stay. +- **Files.** `--exclude GLOB`, or `lint.exclude` in the config file. In a + glob, `*` is any text, a `/` too. +- **A rule, for the project.** `--disable RULE`, or `lint.disable` in the + config file. A rule id that does not exist is an error. + +## Config file + +`cup.config.json`, in the directory that the command runs from. The linter +reads the `lint` section and no other. + +```json +{"lint": {"paths": ["src"], "exclude": ["*/generated/*"], "disable": ["CUP-I005"]}} +``` + +The file is used only when no paths are given on the command line. A key in +`lint` that the linter does not read is an error. + +## Limits + +- A name is followed to its declaration in the same file when that is a + `const`, a function or a class. A `let`, a parameter, an import and the + result of a call are not followed, so a function that returns an anonymous + step is not seen. +- The linter reads syntax only. It does not use the types of the program. +- Test files (`*.test.ts`, `*.spec.ts`, and files under `tests/`) are not + checked by CUP-I005, CUP-I006 and CUP-I007. + +## Development + +The three TypeScript packages are one npm workspace. Run `npm ci` once, at +the root of the repository. Then, in this directory: + +```bash +npm test # builds the core package first, then runs the tests +npm run typecheck +npm run build # JavaScript and type files in dist/ +npm run lint # the linter on its own source, the core package and the conformance runner +``` + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-lint-ts/package.json b/packages/codeupipe-lint-ts/package.json new file mode 100644 index 0000000..eeb7cee --- /dev/null +++ b/packages/codeupipe-lint-ts/package.json @@ -0,0 +1,54 @@ +{ + "name": "codeupipe-lint", + "version": "0.1.0", + "private": true, + "description": "A linter for TypeScript code that uses codeupipe.", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/orchestrate-solutions/codeupipe.git", + "directory": "packages/codeupipe-lint-ts" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "codeupipe-lint": "./dist/cli.js" + }, + "files": [ + "dist", + "LICENSE", + "NOTICE", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=20" + }, + "scripts": { + "core": "npm --prefix ../codeupipe-ts run build", + "pretest": "npm run core", + "test": "node --test \"tests/*.test.ts\"", + "pretypecheck": "npm run core", + "typecheck": "tsc --noEmit", + "prebuild": "npm run core", + "build": "tsc -p tsconfig.build.json", + "lint": "node src/cli.ts src ../codeupipe-ts/src ../codeupipe-ts/examples ../codeupipe-compile-ts/src ../../conformance/runners/typescript", + "prepack": "npm run build" + }, + "dependencies": { + "codeupipe": "0.1.0", + "typescript": "^5.9.0" + }, + "devDependencies": { + "@types/node": "^24.0.0" + }, + "keywords": [ + "pipeline", + "filter", + "payload", + "composition" + ] +} diff --git a/packages/codeupipe-lint-ts/src/analyze-outside-writes-filter.ts b/packages/codeupipe-lint-ts/src/analyze-outside-writes-filter.ts new file mode 100644 index 0000000..8ca3c9b --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-outside-writes-filter.ts @@ -0,0 +1,73 @@ +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { methodsOf } from "./primitive-kind.ts"; +import { WRITE_OUTSIDE_FILTER, makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; + +const READER_METHODS = ["observe", "test", "before", "after", "onError"]; +const WRITES = new Set(["insert", "remove", "merge", "withTrace"]); + +/** The parameter that holds the Payload: the one named `payload`, else the last one. */ +function payloadParameter(method: ts.MethodDeclaration): string | undefined { + const names = method.parameters.map((parameter) => parameter.name.getText()); + return names.includes("payload") ? "payload" : names[names.length - 1]; +} + +/** + * CUP-I004: a write to the Payload from a method that only reads it. + * + * Only a Filter writes. A Tap (`observe`), a Predicate (`test`) and a Hook + * (`before`, `after`, `onError`) read. The runtime does not check, so this + * rule is the guard. Flagged: `insert`, `remove`, `merge` or `withTrace` on + * the Payload parameter of such a method; a chain is one finding. Not + * flagged: a write to `payload.copy()`, and the same method names on any + * other object. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with CUP-I004 findings appended. + */ +export class AnalyzeOutsideWritesFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + const inMethod = (name: string, method: ts.MethodDeclaration) => { + const parameter = payloadParameter(method); + if (parameter === undefined || method.body === undefined) return; + const writes = (node: ts.Node): node is ts.CallExpression => { + if (!ts.isCallExpression(node) || !ts.isPropertyAccessExpression(node.expression) + || !WRITES.has(node.expression.name.text)) { + return false; + } + const owner = node.expression.expression; + return ts.isIdentifier(owner) ? owner.text === parameter : writes(owner); + }; + const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node) || ts.isClassLike(node)) return; // its own scope + if (writes(node)) { + const write = (node.expression as ts.PropertyAccessExpression).name.text; + findings.push(makeFinding(source, node, WRITE_OUTSIDE_FILTER, + `\`${name}\` writes to the Payload (\`${parameter}.${write}\`)`)); + node.arguments.forEach(visit); // the chain is one finding; its arguments are still checked + return; + } + ts.forEachChild(node, visit); + }; + ts.forEachChild(method.body, visit); + }; + const visit = (node: ts.Node) => { + if (ts.isClassLike(node)) { + const methods = methodsOf(node); + for (const name of READER_METHODS) { + const method = methods.get(name); + if (method !== undefined) inMethod(name, method); + } + } + ts.forEachChild(node, visit); + }; + visit(source.tree); + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-plain-logic-filter.ts b/packages/codeupipe-lint-ts/src/analyze-plain-logic-filter.ts new file mode 100644 index 0000000..f037ef8 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-plain-logic-filter.ts @@ -0,0 +1,92 @@ +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { isTestFile, primitiveKind } from "./primitive-kind.ts"; +import { LOGIC_OUTSIDE_PRIMITIVE, makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; + +/** A decision or a loop: the logic that a pipeline is for. */ +function isLogic(node: ts.Node): boolean { + return ts.isIfStatement(node) || ts.isConditionalExpression(node) || ts.isSwitchStatement(node) + || ts.isForStatement(node) || ts.isForInStatement(node) || ts.isForOfStatement(node) + || ts.isWhileStatement(node) || ts.isDoStatement(node) || ts.isTryStatement(node); +} + +function hasLogic(node: ts.Node): boolean { + let found = false; + const visit = (inner: ts.Node) => { + if (found) return; + if (isLogic(inner)) { + found = true; + return; + } + ts.forEachChild(inner, visit); + }; + visit(node); + return found; +} + +const WHAT = "has logic and is not a Filter, a Predicate, a Tap, a Hook or a Pipeline"; + +/** + * CUP-I008, a warning: logic that is not in a primitive. + * + * A project that uses codeupipe writes its logic as Filters, Predicates, + * Taps, Hooks and Pipelines. Logic in a plain function or a plain class works + * as well, so nothing else shows the mistake. This rule shows it. + * + * Logic is a decision or a loop: `if`, `? :`, `switch`, `for`, `while`, + * `do`, `try`. Flagged, once each: a function at the top of a file, a class + * at the top of a file that is not a primitive, and the code at the top of a + * file itself. Not flagged: a primitive (its logic is where it must be), a + * function or a class with no logic, a type, and a test file. Not seen: a + * loop that is a method call (`map`, `filter`, `forEach`), and `&&` / `||`. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with CUP-I008 warnings appended. + */ +export class AnalyzePlainLogicFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + if (isTestFile(source.path)) continue; + let moduleCode: ts.Node | undefined; + for (const statement of source.tree.statements) { + if (ts.isFunctionDeclaration(statement)) { + if (statement.body !== undefined && hasLogic(statement.body)) { + findings.push(makeFinding(source, statement.name ?? statement, LOGIC_OUTSIDE_PRIMITIVE, + `Function \`${statement.name?.text ?? "default"}\` ${WHAT}`)); + } + } else if (ts.isClassDeclaration(statement)) { + if (primitiveKind(statement) === undefined && hasLogic(statement)) { + findings.push(makeFinding(source, statement.name ?? statement, LOGIC_OUTSIDE_PRIMITIVE, + `Class \`${statement.name?.text ?? "default"}\` ${WHAT}`)); + } + } else if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) { + const value = declaration.initializer; + if (value === undefined) continue; + if (ts.isArrowFunction(value) || ts.isFunctionExpression(value)) { + if (hasLogic(value.body)) { + findings.push(makeFinding(source, declaration.name, LOGIC_OUTSIDE_PRIMITIVE, + `Function \`${declaration.name.getText()}\` ${WHAT}`)); + } + } else if (moduleCode === undefined && hasLogic(value)) { + moduleCode = declaration; + } + } + } else if (ts.isInterfaceDeclaration(statement) || ts.isTypeAliasDeclaration(statement) + || ts.isImportDeclaration(statement) || ts.isExportDeclaration(statement)) { + // no logic + } else if (moduleCode === undefined && (isLogic(statement) || hasLogic(statement))) { + moduleCode = statement; + } + } + if (moduleCode !== undefined) { + findings.push(makeFinding(source, moduleCode, LOGIC_OUTSIDE_PRIMITIVE, `The code at the top of this file ${WHAT}`)); + } + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-primitive-names-filter.ts b/packages/codeupipe-lint-ts/src/analyze-primitive-names-filter.ts new file mode 100644 index 0000000..fc57791 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-primitive-names-filter.ts @@ -0,0 +1,54 @@ +import { basename } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; + +import type { Finding } from "./finding.ts"; +import { isTestFile, topLevelPrimitives } from "./primitive-kind.ts"; +import { PRIMITIVE_FILE_NAME, PRIMITIVE_ROLE_SUFFIX, makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; + +/** `ParseHTMLFilter` gives `parse-html-filter`. */ +export function kebabCase(className: string): string { + return className.replace(/(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/g, "-").toLowerCase(); +} + +/** + * CUP-I007: the class name of a primitive ends with its role + * (`DiscountFilter`, `AuditTap`, `PrintStepsHook`, `AtLeastPredicate`, + * `PricingPipeline`). + * + * CUP-I006: the file name is the class name in kebab-case + * (`discount-filter.ts`). + * + * Checked: primitive classes at the top level of a file. Not checked: plain + * classes, and test files. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with CUP-I006 and CUP-I007 findings appended. + */ +export class AnalyzePrimitiveNamesFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + if (isTestFile(source.path)) continue; + const file = basename(source.path); + const dot = file.indexOf("."); + const stem = dot === -1 ? file : file.slice(0, dot); + const extension = dot === -1 ? "" : file.slice(dot); + for (const { node, name, kind } of topLevelPrimitives(source.tree)) { + const at = node.name ?? node; + let wanted = name; + if (!name.endsWith(kind)) { + wanted = name + kind; + findings.push(makeFinding(source, at, PRIMITIVE_ROLE_SUFFIX, + `${kind} \`${name}\` does not say what it is. Name it \`${wanted}\``)); + } + if (stem !== kebabCase(wanted)) { + findings.push(makeFinding(source, at, PRIMITIVE_FILE_NAME, + `\`${wanted}\` is in \`${file}\`. Put it in \`${kebabCase(wanted)}${extension}\``)); + } + } + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-primitives-per-file-filter.ts b/packages/codeupipe-lint-ts/src/analyze-primitives-per-file-filter.ts new file mode 100644 index 0000000..bdbba76 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-primitives-per-file-filter.ts @@ -0,0 +1,32 @@ +import type { Filter, Payload } from "codeupipe"; + +import type { Finding } from "./finding.ts"; +import { isTestFile, topLevelPrimitives } from "./primitive-kind.ts"; +import { ONE_PRIMITIVE_PER_FILE, makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; + +/** + * CUP-I005: one primitive in each file. + * + * Flagged: each primitive class at the top level of a file after the first. + * Not flagged: plain classes and functions beside the primitive, classes in + * a function, and test files. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with CUP-I005 findings appended. + */ +export class AnalyzePrimitivesPerFileFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + if (isTestFile(source.path)) continue; + const [first, ...rest] = topLevelPrimitives(source.tree); + if (first === undefined) continue; + for (const other of rest) { + findings.push(makeFinding(source, other.node.name ?? other.node, ONE_PRIMITIVE_PER_FILE, + `${other.kind} \`${other.name}\` shares this file with ${first.kind} \`${first.name}\``)); + } + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-stale-markers-filter.ts b/packages/codeupipe-lint-ts/src/analyze-stale-markers-filter.ts new file mode 100644 index 0000000..b162ef9 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-stale-markers-filter.ts @@ -0,0 +1,67 @@ +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { fixOf, MARKER_ALLOWS_NOTHING, RULES } from "./rules.ts"; +import type { Source } from "./source.ts"; + +const MARKER = /^cup:\s*(allow-[\w-]*)/; + +/** + * Finds each `cup: allow-...` comment that allowed no finding (CUP-I009, a + * warning). Such a comment stays when the code that it was for is gone or + * has moved, or when its kind has a spelling error. + * + * Only a comment that starts with the marker counts, so text that names a + * marker in a string or in the middle of a sentence is not reported. + * + * Input keys: `findings`, `sources`, `usedMarkers` (from ApplySuppressionsFilter). + * Output keys: `findings`. + */ +export class AnalyzeStaleMarkersFilter implements Filter { + call(payload: Payload): Payload { + const used = new Set(payload.get("usedMarkers") ?? []); + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + for (const marker of this.markersOf(source)) { + if (used.has(`${source.path}:${marker.line}`)) continue; + findings.push({ + path: source.path, line: marker.line, col: marker.col, rule: MARKER_ALLOWS_NOTHING, slug: RULES[MARKER_ALLOWS_NOTHING] ?? "", + message: `The comment \`cup: ${marker.kind}\` allows nothing. ${fixOf(MARKER_ALLOWS_NOTHING)}`, severity: "warning", + }); + } + } + findings.sort((a, b) => a.path.localeCompare(b.path) || a.line - b.line || a.col - b.col); + return payload.insert("findings", findings); + } + + /** Each line of a comment of the file that starts with a marker. */ + markersOf(source: Source): Array<{ line: number; col: number; kind: string }> { + const ranges = new Map(); + const visit = (node: ts.Node) => { + const found = [ + ...(ts.getLeadingCommentRanges(source.text, node.getFullStart()) ?? []), + ...(ts.getTrailingCommentRanges(source.text, node.getEnd()) ?? []), + ]; + for (const range of found) ranges.set(range.pos, range); + // The text of a string or a template can look like a comment. It has no tokens to look in. + if (!ts.isStringLiteralLike(node) && !ts.isTemplateExpression(node)) node.getChildren(source.tree).forEach(visit); + }; + visit(source.tree); + + const markers: Array<{ line: number; col: number; kind: string }> = []; + for (const range of [...ranges.values()].sort((a, b) => a.pos - b.pos)) { + let at = range.pos; + for (const text of source.text.slice(range.pos, range.end).split("\n")) { + const lead = /^\s*(\/\/+|\/\*+|\*(?!\/))?\s*/.exec(text)?.[0].length ?? 0; + const kind = MARKER.exec(text.slice(lead))?.[1]; + if (kind !== undefined) { + const place = source.tree.getLineAndCharacterOfPosition(at + lead); + markers.push({ line: place.line + 1, col: place.character + 1, kind }); + } + at += text.length + 1; + } + } + return markers; + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-state-reads-filter.ts b/packages/codeupipe-lint-ts/src/analyze-state-reads-filter.ts new file mode 100644 index 0000000..230b179 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-state-reads-filter.ts @@ -0,0 +1,111 @@ +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { methodsOf } from "./primitive-kind.ts"; +import { STATE_READ, makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; + +const STEP_METHODS = ["call", "test"]; +const STATE_MEMBERS = new Set(["executed", "skipped", "errors", "timings", "markExecuted", "markSkipped", + "recordTiming", "recordError"]); + +function isThis(node: ts.Node): boolean { + return node.kind === ts.SyntaxKind.ThisKeyword; +} + +function isStateType(type: ts.TypeNode | undefined): boolean { + return type !== undefined && ts.isTypeReferenceNode(type) && type.typeName.getText().split(".").pop() === "State"; +} + +function namesAPipeline(node: ts.Expression): boolean { + const name = ts.isIdentifier(node) ? node.text : ts.isPropertyAccessExpression(node) ? node.name.text : ""; + return name.toLowerCase().includes("pipeline"); +} + +/** The properties of a class that hold a State: by their type, or by what the constructor gives them. */ +function stateHandles(node: ts.ClassLikeDeclaration): Set { + const handles = new Set(); + for (const member of node.members) { + if (ts.isPropertyDeclaration(member) && ts.isIdentifier(member.name)) { + const start = member.initializer; + if (isStateType(member.type) || (start !== undefined && ts.isNewExpression(start) && start.expression.getText() === "State")) { + handles.add(member.name.text); + } + } + if (!ts.isConstructorDeclaration(member) || member.body === undefined) continue; + const stateParameters = new Set(member.parameters.filter((p) => isStateType(p.type)).map((p) => p.name.getText())); + const visit = (inner: ts.Node) => { + if (ts.isBinaryExpression(inner) && inner.operatorToken.kind === ts.SyntaxKind.EqualsToken + && ts.isPropertyAccessExpression(inner.left) && isThis(inner.left.expression)) { + const value = inner.right; + const isState = (ts.isIdentifier(value) && stateParameters.has(value.text)) + || (ts.isPropertyAccessExpression(value) && value.name.text === "state") + || (ts.isNewExpression(value) && value.expression.getText() === "State"); + if (isState) handles.add(inner.left.name.text); + } + ts.forEachChild(inner, visit); + }; + visit(member.body); + } + return handles; +} + +/** + * CUP-I003: a Filter (`call`) or a Predicate (`test`) that reads the State + * of a pipeline. + * + * Flagged: `.state.` where the member is one of State's own, + * `.state` when the name says it is a pipeline, and a property + * that holds a State. Not flagged: `this.state` (the step's own property), a + * Payload key named "state", and anything in a Tap or a Hook. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with CUP-I003 findings appended. + */ +export class AnalyzeStateReadsFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + const inClass = (node: ts.ClassLikeDeclaration) => { + const handles = stateHandles(node); + const methods = methodsOf(node); + for (const name of STEP_METHODS) { + const body = methods.get(name)?.body; + if (body !== undefined) inBody(body, handles); + } + }; + const reachesState = (node: ts.PropertyAccessExpression, handles: Set): boolean => { + let outer: string | undefined; + let at: ts.Expression = node; + while (ts.isPropertyAccessExpression(at)) { + if (isThis(at.expression) && handles.has(at.name.text)) return true; + if (at.name.text === "state" && !isThis(at.expression) + && ((outer !== undefined && STATE_MEMBERS.has(outer)) || namesAPipeline(at.expression))) { + return true; + } + outer = at.name.text; + at = at.expression; + } + return false; + }; + const inBody = (body: ts.Node, handles: Set) => { + const visit = (node: ts.Node) => { + if (ts.isFunctionLike(node) || ts.isClassLike(node)) return; // its own scope + if (ts.isPropertyAccessExpression(node) && reachesState(node, handles)) { + findings.push(makeFinding(source, node, STATE_READ, `Step reads pipeline State (\`${node.getText()}\`)`)); + return; + } + ts.forEachChild(node, visit); + }; + ts.forEachChild(body, visit); + }; + const visit = (node: ts.Node) => { + if (ts.isClassLike(node)) inClass(node); + ts.forEachChild(node, visit); + }; + visit(source.tree); + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/analyze-steps-filter.ts b/packages/codeupipe-lint-ts/src/analyze-steps-filter.ts new file mode 100644 index 0000000..66ec5a4 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/analyze-steps-filter.ts @@ -0,0 +1,45 @@ +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { makeFinding } from "./rules.ts"; +import type { Source } from "./source.ts"; +import { classifyStep } from "./step-classifier.ts"; +import type { Role } from "./step-classifier.ts"; + +const STEP_METHODS: Record = { addFilter: "Filter", addTap: "Tap", addPipeline: "Pipeline" }; +const VALVE_ROLES: Role[] = ["Valve predicate", "Valve filter"]; + +/** + * CUP-A001, A002, A003, A005, A008, A009: a step that has no name. + * + * A step is the argument of `addFilter`, `addTap` or `addPipeline` (on any + * receiver), or one of the two arguments of `addValve(predicate, filter)`. + * + * Input keys: `sources`, `findings`. + * Output keys: `findings`, with the new findings appended. + */ +export class AnalyzeStepsFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + for (const source of payload.get("sources") ?? []) { + const visit = (node: ts.Node) => { + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression)) { + const method = node.expression.name.text; + const roles = method === "addValve" ? VALVE_ROLES + : STEP_METHODS[method] !== undefined ? [STEP_METHODS[method]] : []; + roles.forEach((role, index) => { + const argument = node.arguments[index]; + const verdict = argument === undefined ? undefined : classifyStep(argument, role); + if (argument !== undefined && verdict !== undefined) { + findings.push(makeFinding(source, argument, verdict.rule, `${role} ${verdict.what}`, node)); + } + }); + } + ts.forEachChild(node, visit); + }; + visit(source.tree); + } + return payload.insert("findings", findings); + } +} diff --git a/packages/codeupipe-lint-ts/src/apply-suppressions-filter.ts b/packages/codeupipe-lint-ts/src/apply-suppressions-filter.ts new file mode 100644 index 0000000..859714f --- /dev/null +++ b/packages/codeupipe-lint-ts/src/apply-suppressions-filter.ts @@ -0,0 +1,44 @@ +import type { Filter, Payload } from "codeupipe"; + +import type { Finding } from "./finding.ts"; +import type { Source } from "./source.ts"; + +const ANONYMOUS = /cup:\s*allow-anonymous\b/; +const INVARIANT = /cup:\s*allow-(state-read|outside-write|shared-file|name|plain)\b/; +const COMMENT_ONLY = /^\s*(\/\/|\/\*|\*)/; + +/** + * Drops the findings that a comment allows. + * + * `// cup: allow-anonymous` allows a CUP-A finding. `// cup: allow-state-read`, + * `allow-outside-write`, `allow-shared-file`, `allow-name` and `allow-plain` + * allow a CUP-I finding. The comment is on the line of the finding, on the line of the + * `add...` call around it, or alone on the line directly above. Any text can + * follow the marker. + * + * Input keys: `findings`, `sources`. + * Output keys: `findings` (kept), `suppressed` (dropped), `usedMarkers` (the + * place of each comment that allowed a finding, as `path:line`). + */ +export class ApplySuppressionsFilter implements Filter { + call(payload: Payload): Payload { + const lines = new Map(); + for (const source of payload.get("sources") ?? []) { + lines.set(source.path, source.text.split(/\r?\n/)); + } + const kept: Finding[] = []; + const suppressed: Finding[] = []; + const used = new Set(); + for (const finding of payload.get("findings") ?? []) { + const marker = finding.rule.startsWith("CUP-A") ? ANONYMOUS : INVARIANT; + const text = lines.get(finding.path) ?? []; + const line = (number: number | undefined) => (number === undefined ? "" : text[number - 1] ?? ""); + const above = line(finding.line - 1); + const places = [finding.line, finding.callLine].filter((number) => marker.test(line(number))); + if (COMMENT_ONLY.test(above) && marker.test(above)) places.push(finding.line - 1); + for (const place of places) used.add(`${finding.path}:${place}`); + (places.length > 0 ? suppressed : kept).push(finding); + } + return payload.insert("findings", kept).insert("suppressed", suppressed).insert("usedMarkers", [...used]); + } +} diff --git a/packages/codeupipe-lint-ts/src/cli.ts b/packages/codeupipe-lint-ts/src/cli.ts new file mode 100644 index 0000000..c85812f --- /dev/null +++ b/packages/codeupipe-lint-ts/src/cli.ts @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * codeupipe-lint [--exclude GLOB] [--disable RULE] [--strict] [--config FILE] [--rules] [paths...] + * + * With no paths, they come from cup.config.json. Exit code: 0 no findings, + * 1 findings, 2 a path could not be read or the arguments cannot be used. + * A warning does not make the exit code 1, unless --strict is given. + */ +import { parseArgs } from "node:util"; + +import { lintPaths } from "./lint-paths.ts"; +import { CONFIG_NAME, readLintConfig } from "./read-config.ts"; +import { RULES } from "./rules.ts"; + +async function main(argv: string[]): Promise { + let args; + try { + args = parseArgs({ + args: argv, allowPositionals: true, + options: { + exclude: { type: "string", multiple: true }, + disable: { type: "string", multiple: true }, + config: { type: "string" }, + rules: { type: "boolean" }, + strict: { type: "boolean" }, + }, + }); + } catch (error) { + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + if (args.values.rules) { + for (const rule of Object.keys(RULES).sort()) console.log(`${rule} ${RULES[rule]}`); + return 0; + } + + let paths = args.positionals; + let exclude = args.values.exclude; + let disable = args.values.disable; + if (paths.length === 0) { + // The file stands in for the arguments only when no paths are given: a + // path that someone names is checked, whatever the file excludes. + try { + const lint = readLintConfig(args.values.config) ?? {}; + paths = lint.paths ?? []; + exclude = exclude ?? lint.exclude; + disable = disable ?? lint.disable; + } catch (error) { + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + } + if (paths.length === 0) { + console.error(`error: give at least one path, or list them under lint.paths in ${CONFIG_NAME}`); + return 2; + } + const unknown = (disable ?? []).filter((rule) => RULES[rule] === undefined); + if (unknown.length > 0) { + console.error(`error: no such rule: ${unknown.join(", ")} (the rules are listed by --rules)`); + return 2; + } + + const out = await lintPaths(paths, { exclude, disable, strict: args.values.strict === true }); + console.log(out.get("report")); + const errorReport = out.get("errorReport"); + if (errorReport) console.error(errorReport); + return out.get("exitCode") ?? 2; +} + +process.exitCode = await main(process.argv.slice(2)); diff --git a/packages/codeupipe-lint-ts/src/collect-files-filter.ts b/packages/codeupipe-lint-ts/src/collect-files-filter.ts new file mode 100644 index 0000000..fcecebd --- /dev/null +++ b/packages/codeupipe-lint-ts/src/collect-files-filter.ts @@ -0,0 +1,53 @@ +import { readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; + +import { fnmatch } from "./fnmatch.ts"; + +const EXTENSIONS = [".ts", ".tsx", ".mts", ".cts"]; +const SKIP_DIRS = new Set([".git", "node_modules", "build", "dist", "coverage"]); + +function isSource(path: string): boolean { + return EXTENSIONS.some((extension) => path.endsWith(extension)) && !/\.d\.[cm]?ts$/.test(path); +} + +/** + * The TypeScript source files under the given paths. + * + * Input keys: `paths` (files or directories), optional `exclude` (globs + * matched against the path of each file). + * Output keys: `files` (paths), `errors` (appended). + * + * Directories are walked. Version control, dependency and build-output + * directories are not read, and nor are `.d.ts` files. + */ +export class CollectFilesFilter implements Filter { + call(payload: Payload): Payload { + const exclude = payload.get("exclude") ?? []; + const errors = [...(payload.get("errors") ?? [])]; + const files: string[] = []; + + const take = (path: string) => { + if (isSource(path) && !exclude.some((pattern) => fnmatch(path, pattern))) files.push(path); + }; + const walk = (directory: string) => { + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : 1))) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) walk(path); + } else if (entry.isFile()) { + take(path); + } + } + }; + + for (const root of payload.get("paths") ?? []) { + const stat = statSync(root, { throwIfNoEntry: false }); + if (stat === undefined) errors.push(`${root}: no such file or directory`); + else if (stat.isDirectory()) walk(root); + else take(root); + } + return payload.insert("files", files).insert("errors", errors); + } +} diff --git a/packages/codeupipe-lint-ts/src/drop-disabled-filter.ts b/packages/codeupipe-lint-ts/src/drop-disabled-filter.ts new file mode 100644 index 0000000..3b998b0 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/drop-disabled-filter.ts @@ -0,0 +1,19 @@ +import type { Filter, Payload } from "codeupipe"; + +import type { Finding } from "./finding.ts"; + +/** + * Drops the findings of the rules that the project switched off, from those + * that are reported and from those that a comment allowed. + * + * Input keys: `findings`, `suppressed`, optional `disable` (rule ids). + * Output keys: `findings`, `suppressed`. + */ +export class DropDisabledFilter implements Filter { + call(payload: Payload): Payload { + const disabled = new Set(payload.get("disable") ?? []); + const on = (finding: Finding) => !disabled.has(finding.rule); + return payload.insert("findings", (payload.get("findings") ?? []).filter(on)) + .insert("suppressed", (payload.get("suppressed") ?? []).filter(on)); + } +} diff --git a/packages/codeupipe-lint-ts/src/finding.ts b/packages/codeupipe-lint-ts/src/finding.ts new file mode 100644 index 0000000..7bfb50e --- /dev/null +++ b/packages/codeupipe-lint-ts/src/finding.ts @@ -0,0 +1,22 @@ +/** + * One finding, at `path:line:col` (line and column start at 1). + * + * `callLine` is the line of the `add...` call around the finding, when it is + * a different line: a comment that suppresses can be on that line too. + */ +export type Finding = { + path: string; + line: number; + col: number; + rule: string; + slug: string; + message: string; + callLine?: number; + /** A warning is reported and does not fail the run, unless the run is strict. */ + severity?: "warning"; +}; + +export function formatFinding(finding: Finding): string { + const warning = finding.severity === "warning" ? "warning: " : ""; + return `${finding.path}:${finding.line}:${finding.col} ${finding.rule} ${finding.slug} ${warning}${finding.message}`; +} diff --git a/packages/codeupipe-lint-ts/src/fnmatch.ts b/packages/codeupipe-lint-ts/src/fnmatch.ts new file mode 100644 index 0000000..5f3c291 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/fnmatch.ts @@ -0,0 +1,28 @@ +/** + * Whether `path` matches `pattern`, with the rules of a shell glob as + * Python's fnmatch has them: `*` is any text (a `/` too), `?` is one + * character, `[abc]` is one of the characters. + */ +export function fnmatch(path: string, pattern: string): boolean { + let expression = ""; + for (let index = 0; index < pattern.length; index++) { + const char = pattern[index] ?? ""; + if (char === "*") { + expression += ".*"; + } else if (char === "?") { + expression += "."; + } else if (char === "[") { + const end = pattern.indexOf("]", index + 1); + if (end === -1) { + expression += "\\["; + } else { + const body = pattern.slice(index + 1, end).replaceAll("\\", "\\\\"); + expression += "[" + (body.startsWith("!") ? "^" + body.slice(1) : body) + "]"; + index = end; + } + } else { + expression += char.replace(/[.+^${}()|\\\]]/g, "\\$&"); + } + } + return new RegExp(`^${expression}$`, "s").test(path); +} diff --git a/packages/codeupipe-lint-ts/src/format-report-filter.ts b/packages/codeupipe-lint-ts/src/format-report-filter.ts new file mode 100644 index 0000000..5c6c2ec --- /dev/null +++ b/packages/codeupipe-lint-ts/src/format-report-filter.ts @@ -0,0 +1,37 @@ +import type { Filter, Payload } from "codeupipe"; + +import { formatFinding } from "./finding.ts"; +import type { Finding } from "./finding.ts"; + +/** + * The findings as sorted report lines, and the exit code. + * + * Input keys: `findings`, `suppressed`, `errors`, optional `strict`. + * Output keys: `findings` (sorted), `report` (text for stdout), + * `errorReport` (text for stderr), `exitCode`: 0 no findings, 1 findings, + * 2 a path could not be read, so the check is not complete. + * + * A warning is in the report and is counted apart. It does not make the + * exit code 1, unless `strict` is true. + */ +export class FormatReportFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])].sort((a, b) => + (a.path < b.path ? -1 : a.path > b.path ? 1 : 0) || a.line - b.line || a.col - b.col + || (a.rule < b.rule ? -1 : a.rule > b.rule ? 1 : 0)); + const suppressed = payload.get("suppressed") ?? []; + const errors = payload.get("errors") ?? []; + const lines = findings.map(formatFinding); + const warnings = findings.filter((finding) => finding.severity === "warning").length; + const real = findings.length - warnings; + let summary = real === 0 ? "No findings" : `${real} ${real === 1 ? "finding" : "findings"}`; + if (suppressed.length > 0) summary += ` (${suppressed.length} suppressed)`; + summary += "."; + if (warnings > 0) summary += ` ${warnings} ${warnings === 1 ? "warning" : "warnings"}.`; + lines.push(summary); + const fails = real > 0 || (warnings > 0 && payload.get("strict") === true); + return payload.insert("findings", findings).insert("report", lines.join("\n")) + .insert("errorReport", errors.map((error) => "error: " + error).join("\n")) + .insert("exitCode", errors.length > 0 ? 2 : fails ? 1 : 0); + } +} diff --git a/packages/codeupipe-lint-ts/src/index.ts b/packages/codeupipe-lint-ts/src/index.ts new file mode 100644 index 0000000..c0e9c9d --- /dev/null +++ b/packages/codeupipe-lint-ts/src/index.ts @@ -0,0 +1,5 @@ +/** codeupipe-lint: a linter for TypeScript code that uses codeupipe. */ +export type { Finding } from "./finding.ts"; +export { lintPaths } from "./lint-paths.ts"; +export { LintPipeline } from "./lint-pipeline.ts"; +export { RULES } from "./rules.ts"; diff --git a/packages/codeupipe-lint-ts/src/lint-paths.ts b/packages/codeupipe-lint-ts/src/lint-paths.ts new file mode 100644 index 0000000..f5af58f --- /dev/null +++ b/packages/codeupipe-lint-ts/src/lint-paths.ts @@ -0,0 +1,13 @@ +import { Payload } from "codeupipe"; + +import { LintPipeline } from "./lint-pipeline.ts"; + +/** + * Run the linter over `paths`. The Payload that comes back holds `findings`, + * `suppressed`, `errors`, `report`, `errorReport` and `exitCode`. + */ +export function lintPaths(paths: string[], options: { exclude?: string[]; disable?: string[]; strict?: boolean } = {}): Promise { + return new LintPipeline().run(new Payload({ + paths, findings: [], errors: [], exclude: options.exclude ?? [], disable: options.disable ?? [], strict: options.strict === true, + })); +} diff --git a/packages/codeupipe-lint-ts/src/lint-pipeline.ts b/packages/codeupipe-lint-ts/src/lint-pipeline.ts new file mode 100644 index 0000000..dc3002f --- /dev/null +++ b/packages/codeupipe-lint-ts/src/lint-pipeline.ts @@ -0,0 +1,33 @@ +import { Pipeline } from "codeupipe"; + +import { AnalyzeOutsideWritesFilter } from "./analyze-outside-writes-filter.ts"; +import { AnalyzePlainLogicFilter } from "./analyze-plain-logic-filter.ts"; +import { AnalyzePrimitiveNamesFilter } from "./analyze-primitive-names-filter.ts"; +import { AnalyzeStaleMarkersFilter } from "./analyze-stale-markers-filter.ts"; +import { AnalyzePrimitivesPerFileFilter } from "./analyze-primitives-per-file-filter.ts"; +import { AnalyzeStateReadsFilter } from "./analyze-state-reads-filter.ts"; +import { AnalyzeStepsFilter } from "./analyze-steps-filter.ts"; +import { ApplySuppressionsFilter } from "./apply-suppressions-filter.ts"; +import { CollectFilesFilter } from "./collect-files-filter.ts"; +import { DropDisabledFilter } from "./drop-disabled-filter.ts"; +import { FormatReportFilter } from "./format-report-filter.ts"; +import { ParseFilesFilter } from "./parse-files-filter.ts"; + +/** The linter, as a codeupipe pipeline. */ +export class LintPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new CollectFilesFilter()); // paths to files + this.addFilter(new ParseFilesFilter()); // files to syntax trees (CUP-I000) + this.addFilter(new AnalyzeStepsFilter()); // CUP-A001, A002, A003, A005, A008, A009 + this.addFilter(new AnalyzeStateReadsFilter()); // CUP-I003 + this.addFilter(new AnalyzeOutsideWritesFilter()); // CUP-I004 + this.addFilter(new AnalyzePrimitivesPerFileFilter()); // CUP-I005 + this.addFilter(new AnalyzePrimitiveNamesFilter()); // CUP-I006, CUP-I007 + this.addFilter(new AnalyzePlainLogicFilter()); // CUP-I008, a warning + this.addFilter(new ApplySuppressionsFilter()); + this.addFilter(new AnalyzeStaleMarkersFilter()); // CUP-I009, a warning + this.addFilter(new DropDisabledFilter()); // last, so that a comment for a rule that is off is not stale + this.addFilter(new FormatReportFilter()); + } +} diff --git a/packages/codeupipe-lint-ts/src/parse-files-filter.ts b/packages/codeupipe-lint-ts/src/parse-files-filter.ts new file mode 100644 index 0000000..3f30eee --- /dev/null +++ b/packages/codeupipe-lint-ts/src/parse-files-filter.ts @@ -0,0 +1,45 @@ +import { readFileSync } from "node:fs"; + +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import type { Finding } from "./finding.ts"; +import { PARSE_ERROR, RULES } from "./rules.ts"; +import type { Source } from "./source.ts"; + +/** + * Reads and parses each file. + * + * Input keys: `files`, `findings`, `errors`. + * Output keys: `sources` (the parsed files), `findings` with CUP-I000 for a + * file that has a syntax error, `errors` for a file that cannot be read. + */ +export class ParseFilesFilter implements Filter { + call(payload: Payload): Payload { + const findings = [...(payload.get("findings") ?? [])]; + const errors = [...(payload.get("errors") ?? [])]; + const sources: Source[] = []; + for (const path of payload.get("files") ?? []) { + let text: string; + try { + text = readFileSync(path, "utf8"); + } catch (error) { + errors.push(`${path}: ${error instanceof Error ? error.message : String(error)}`); + continue; + } + const tree = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true); + const problem = ts.transpileModule(text, { fileName: path, reportDiagnostics: true }).diagnostics?.[0]; + if (problem !== undefined) { + const at = tree.getLineAndCharacterOfPosition(problem.start ?? 0); + const what = ts.flattenDiagnosticMessageText(problem.messageText, " "); + findings.push({ + path, line: at.line + 1, col: at.character + 1, rule: PARSE_ERROR, slug: RULES[PARSE_ERROR] ?? "", + message: `${what} Fix the syntax error so the file can be checked.`, + }); + continue; + } + sources.push({ path, text, tree }); + } + return payload.insert("sources", sources).insert("findings", findings).insert("errors", errors); + } +} diff --git a/packages/codeupipe-lint-ts/src/primitive-kind.ts b/packages/codeupipe-lint-ts/src/primitive-kind.ts new file mode 100644 index 0000000..6613e77 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/primitive-kind.ts @@ -0,0 +1,84 @@ +import ts from "typescript"; + +export type PrimitiveKind = "Pipeline" | "Hook" | "Filter" | "Tap" | "Predicate"; + +const BY_METHOD: Array<[string, PrimitiveKind]> = [["call", "Filter"], ["observe", "Tap"], ["test", "Predicate"]]; +const BY_INTERFACE: Record = { Filter: "Filter", Tap: "Tap", Predicate: "Predicate" }; +const HOOK_METHODS = ["before", "after", "onError"]; + +function heritage(node: ts.ClassLikeDeclaration, token: ts.SyntaxKind): string[] { + const names: string[] = []; + for (const clause of node.heritageClauses ?? []) { + if (clause.token !== token) continue; + for (const type of clause.types) { + const expression = type.expression; + if (ts.isIdentifier(expression)) names.push(expression.text); + else if (ts.isPropertyAccessExpression(expression)) names.push(expression.name.text); + } + } + return names; +} + +/** The methods of a class, by name. */ +export function methodsOf(node: ts.ClassLikeDeclaration): Map { + const methods = new Map(); + for (const member of node.members) { + if (ts.isMethodDeclaration(member) && ts.isIdentifier(member.name)) methods.set(member.name.text, member); + } + return methods; +} + +/** One parameter that the caller must give: not optional, no default, not a rest parameter. */ +function takesOnlyThePayload(method: ts.MethodDeclaration): boolean { + const [parameter, ...others] = method.parameters; + return parameter !== undefined && others.length === 0 && parameter.dotDotDotToken === undefined + && parameter.questionToken === undefined && parameter.initializer === undefined; +} + +/** + * Which primitive a class is, read from its shape. Undefined for any other class. + * + * - `extends Pipeline` is a Pipeline. + * - `extends Hook`, or the methods `before`, `after` and `onError`, is a Hook. + * - `implements Filter`, or `call` with one parameter, is a Filter. + * - `implements Tap`, or `observe` with one parameter, is a Tap. + * - `implements Predicate`, or `test` with one parameter, is a Predicate. + * + * `call`, `observe` and `test` count only with exactly one required + * parameter, so an unrelated `call(a, b)` or `observe(options = {})` is not + * mistaken for a step. + */ +export function primitiveKind(node: ts.ClassLikeDeclaration): PrimitiveKind | undefined { + const methods = methodsOf(node); + const bases = heritage(node, ts.SyntaxKind.ExtendsKeyword); + if (bases.includes("Pipeline")) return "Pipeline"; + if (bases.includes("Hook") || HOOK_METHODS.every((name) => methods.has(name))) return "Hook"; + for (const name of heritage(node, ts.SyntaxKind.ImplementsKeyword)) { + const kind = BY_INTERFACE[name]; + if (kind !== undefined) return kind; + } + for (const [name, kind] of BY_METHOD) { + const method = methods.get(name); + if (method !== undefined && takesOnlyThePayload(method)) return kind; + } + return undefined; +} + +/** A test file, or a file under a `tests` directory. */ +export function isTestFile(path: string): boolean { + const parts = path.replaceAll("\\", "/").split("/"); + const name = parts[parts.length - 1] ?? ""; + return parts.slice(0, -1).some((part) => part === "tests" || part === "__tests__") + || /\.(test|spec)\.[cm]?tsx?$/.test(name); +} + +/** The classes at the top level of a file that are primitives. */ +export function topLevelPrimitives(tree: ts.SourceFile): Array<{ node: ts.ClassDeclaration; name: string; kind: PrimitiveKind }> { + const found: Array<{ node: ts.ClassDeclaration; name: string; kind: PrimitiveKind }> = []; + for (const statement of tree.statements) { + if (!ts.isClassDeclaration(statement) || statement.name === undefined) continue; + const kind = primitiveKind(statement); + if (kind !== undefined) found.push({ node: statement, name: statement.name.text, kind }); + } + return found; +} diff --git a/packages/codeupipe-lint-ts/src/read-config.ts b/packages/codeupipe-lint-ts/src/read-config.ts new file mode 100644 index 0000000..17bddcc --- /dev/null +++ b/packages/codeupipe-lint-ts/src/read-config.ts @@ -0,0 +1,43 @@ +import { existsSync, readFileSync } from "node:fs"; + +export const CONFIG_NAME = "cup.config.json"; +const LINT_KEYS = ["paths", "exclude", "disable"]; + +export type LintConfig = { paths?: string[]; exclude?: string[]; disable?: string[] }; + +/** + * The `lint` section of the config file, checked. Undefined when the file + * is not there. With `path` given, a file that is not there is an error. + * + * A key that the linter does not read is an error, so the file cannot hold a + * setting that does nothing. Other sections of the file are for other tools. + */ +export function readLintConfig(path?: string): LintConfig | undefined { + const name = path ?? CONFIG_NAME; + if (!existsSync(name)) { + if (path === undefined) return undefined; + throw new Error(`${name}: no such file`); + } + let config: unknown; + try { + config = JSON.parse(readFileSync(name, "utf8")); + } catch (error) { + throw new Error(`${name} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); + } + if (typeof config !== "object" || config === null || Array.isArray(config)) { + throw new Error(`${name} must hold a JSON object`); + } + const lint = (config as Record).lint ?? {}; + if (typeof lint !== "object" || lint === null || Array.isArray(lint)) { + throw new Error(`${name}: lint must be an object`); + } + for (const [key, value] of Object.entries(lint)) { + if (!LINT_KEYS.includes(key)) { + throw new Error(`${name}: lint: unknown key '${key}' (the keys are ${LINT_KEYS.join(", ")})`); + } + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { + throw new Error(`${name}: lint.${key} must be a list of strings`); + } + } + return lint as LintConfig; +} diff --git a/packages/codeupipe-lint-ts/src/rules.ts b/packages/codeupipe-lint-ts/src/rules.ts new file mode 100644 index 0000000..f8e15f3 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/rules.ts @@ -0,0 +1,85 @@ +import type { Finding } from "./finding.ts"; +import { position } from "./source.ts"; +import type { Source } from "./source.ts"; +import type ts from "typescript"; + +export const PARSE_ERROR = "CUP-I000"; +export const FUNCTION_STEP = "CUP-A001"; +export const NAMED_FUNCTION_STEP = "CUP-A002"; +export const LOCAL_TYPE_STEP = "CUP-A003"; +export const UNNAMED_PIPELINE = "CUP-A005"; +export const FUNCTION_PREDICATE = "CUP-A008"; +export const OBJECT_LITERAL_STEP = "CUP-A009"; +export const STATE_READ = "CUP-I003"; +export const WRITE_OUTSIDE_FILTER = "CUP-I004"; +export const ONE_PRIMITIVE_PER_FILE = "CUP-I005"; +export const PRIMITIVE_FILE_NAME = "CUP-I006"; +export const PRIMITIVE_ROLE_SUFFIX = "CUP-I007"; +export const LOGIC_OUTSIDE_PRIMITIVE = "CUP-I008"; +export const MARKER_ALLOWS_NOTHING = "CUP-I009"; + +/** The rules whose findings are warnings. */ +export const WARNINGS = new Set([LOGIC_OUTSIDE_PRIMITIVE, MARKER_ALLOWS_NOTHING]); + +/** Rule id to its short name. The ids and names are those of the Python linter, where it has the rule. */ +export const RULES: Record = { + [PARSE_ERROR]: "parse-error", + [FUNCTION_STEP]: "lambda-step", + [NAMED_FUNCTION_STEP]: "local-function-step", + [LOCAL_TYPE_STEP]: "local-type-step", + [UNNAMED_PIPELINE]: "unnamed-pipeline-step", + [FUNCTION_PREDICATE]: "function-predicate", + [OBJECT_LITERAL_STEP]: "object-literal-step", + [STATE_READ]: "state-read-in-step", + [WRITE_OUTSIDE_FILTER]: "write-outside-filter", + [ONE_PRIMITIVE_PER_FILE]: "one-primitive-per-file", + [PRIMITIVE_FILE_NAME]: "primitive-file-name", + [PRIMITIVE_ROLE_SUFFIX]: "primitive-role-suffix", + [LOGIC_OUTSIDE_PRIMITIVE]: "logic-outside-primitive", + [MARKER_ALLOWS_NOTHING]: "marker-allows-nothing", +}; + +const NAMED_CLASS = "Define it at module level as a named class (a Valve predicate: a class with test(payload)) " + + "so it has a name in State and can be tested on its own."; + +const FIX: Record = { + [PARSE_ERROR]: "Fix the syntax error so the file can be checked.", + [FUNCTION_STEP]: NAMED_CLASS, + [NAMED_FUNCTION_STEP]: NAMED_CLASS, + [LOCAL_TYPE_STEP]: NAMED_CLASS, + [OBJECT_LITERAL_STEP]: NAMED_CLASS, + [UNNAMED_PIPELINE]: "A nested pipeline must be a named pipeline type (class UserOnboarding extends Pipeline), " + + "named like a filter. A plain Pipeline can only run at the top level.", + [FUNCTION_PREDICATE]: "Valve predicates are objects (conformance/API.md, \"Valve\"): wrap the condition in a " + + "named class with test(payload), configuration on the instance, and pass an instance.", + [STATE_READ]: "A Filter's result and a Valve's decision must depend only on the Payload. Put what the step " + + "needs into the Payload, or observe State from a Tap or a Hook.", + [WRITE_OUTSIDE_FILTER]: "Only a Filter writes to the Payload, and nothing at run time stops this write. " + + "Put the write in a Filter, or write to payload.copy().", + [ONE_PRIMITIVE_PER_FILE]: "Give each Filter, Predicate, Tap, Hook and named Pipeline its own file, so it can " + + "be tested alone and used again. The pipeline file is where they come together.", + [PRIMITIVE_FILE_NAME]: "The file name of a primitive is its class name in kebab-case, so one name gives the other.", + [PRIMITIVE_ROLE_SUFFIX]: "A primitive's class name ends with its role (Filter, Tap, Hook, Predicate, " + + "Pipeline), so the name tells what the class is.", + [LOGIC_OUTSIDE_PRIMITIVE]: "Write it as a primitive, so that the pipeline has the logic and it can be tested as a step. " + + "For logic that must be plain code, say why: // cup: allow-plain ", + [MARKER_ALLOWS_NOTHING]: "Remove it, or put it on the line of the finding that it is for. A comment that allows nothing " + + "says that the code has an exception that it does not have.", +}; + +/** What to do about a finding of `rule`. */ +export function fixOf(rule: string): string { + return FIX[rule] ?? ""; +} + +/** A Finding at `node`. Its message reads `. `. */ +export function makeFinding(source: Source, node: ts.Node, rule: string, what: string, callNode?: ts.Node): Finding { + const { line, col } = position(source, node); + const finding: Finding = { path: source.path, line, col, rule, slug: RULES[rule] ?? "", message: `${what}. ${FIX[rule]}` }; + if (WARNINGS.has(rule)) finding.severity = "warning"; + if (callNode !== undefined) { + const callLine = position(source, callNode).line; + if (callLine !== line) finding.callLine = callLine; + } + return finding; +} diff --git a/packages/codeupipe-lint-ts/src/source.ts b/packages/codeupipe-lint-ts/src/source.ts new file mode 100644 index 0000000..f542e71 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/source.ts @@ -0,0 +1,10 @@ +import ts from "typescript"; + +/** One parsed file. */ +export type Source = { path: string; text: string; tree: ts.SourceFile }; + +/** The line and column of a node, from 1. */ +export function position(source: Source, node: ts.Node): { line: number; col: number } { + const at = source.tree.getLineAndCharacterOfPosition(node.getStart(source.tree)); + return { line: at.line + 1, col: at.character + 1 }; +} diff --git a/packages/codeupipe-lint-ts/src/step-classifier.ts b/packages/codeupipe-lint-ts/src/step-classifier.ts new file mode 100644 index 0000000..bb95d23 --- /dev/null +++ b/packages/codeupipe-lint-ts/src/step-classifier.ts @@ -0,0 +1,125 @@ +import ts from "typescript"; + +import { + FUNCTION_PREDICATE, FUNCTION_STEP, LOCAL_TYPE_STEP, NAMED_FUNCTION_STEP, OBJECT_LITERAL_STEP, UNNAMED_PIPELINE, +} from "./rules.ts"; + +export type Role = "Filter" | "Tap" | "Pipeline" | "Valve predicate" | "Valve filter"; + +/** A rule, and the words that say what the value is: `is a function`. */ +export type Verdict = { rule: string; what: string }; + +function unwrap(node: ts.Expression): ts.Expression { + let inner = node; + while (ts.isParenthesizedExpression(inner) || ts.isAsExpression(inner) || ts.isNonNullExpression(inner) + || ts.isSatisfiesExpression(inner)) { + inner = inner.expression; + } + return inner; +} + +function isScope(node: ts.Node): boolean { + return ts.isSourceFile(node) || ts.isFunctionLike(node) || ts.isModuleBlock(node) || ts.isClassStaticBlockDeclaration(node); +} + +/** The declarations of `name` that the scope `scope` holds itself, not in a function inside it. */ +function declarationsIn(scope: ts.Node, name: string): ts.Node[] { + const found: ts.Node[] = []; + if (ts.isFunctionLike(scope)) { + for (const parameter of scope.parameters) { + if (parameter.getText().includes(name)) found.push(parameter); + } + } + const visit = (node: ts.Node) => { + if (ts.isVariableDeclaration(node) || ts.isBindingElement(node)) { + if (ts.isIdentifier(node.name) ? node.name.text === name : node.name.getText().includes(name)) found.push(node); + } else if ((ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node)) && node.name?.text === name) { + found.push(node); + } else if ((ts.isImportClause(node) || ts.isImportSpecifier(node) || ts.isNamespaceImport(node)) + && node.name?.text === name) { + found.push(node); + } else if (ts.isCatchClause(node) && node.variableDeclaration?.name.getText() === name) { + found.push(node); + } + if (node !== scope && isScope(node)) return; + ts.forEachChild(node, visit); + }; + ts.forEachChild(scope, visit); + return found; +} + +/** + * The one declaration that `name` has at `from`, by the rules of scope: the + * nearest function outwards to the file. Undefined when the file does not + * bind the name, or binds it in a way that this linter does not follow. + */ +function resolve(name: ts.Identifier): ts.Node | undefined { + for (let scope: ts.Node | undefined = name.parent; scope !== undefined; scope = scope.parent) { + if (!isScope(scope)) continue; + const found = declarationsIn(scope, name.text); + if (found.length === 1) return found[0]; + if (found.length > 1) return undefined; + } + return undefined; +} + +function isInsideFunction(node: ts.Node): boolean { + for (let parent = node.parent; parent !== undefined; parent = parent.parent) { + if (ts.isFunctionLike(parent)) return true; + } + return false; +} + +function isConst(declaration: ts.VariableDeclaration): boolean { + return ts.isVariableDeclarationList(declaration.parent) && (declaration.parent.flags & ts.NodeFlags.Const) !== 0; +} + +/** + * What is wrong with a value that is given as a step, or undefined. + * + * A name is followed to its declaration in the same file when that is a + * `const`, a function or a class. A `let`, a parameter, an import and the + * result of a call are not followed. + */ +export function classifyStep(expression: ts.Expression, role: Role, seen: Set = new Set()): Verdict | undefined { + const node = unwrap(expression); + if (seen.has(node)) return undefined; + seen.add(node); + + if (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) { + return { rule: FUNCTION_STEP, what: "is a function" }; + } + if (ts.isObjectLiteralExpression(node)) { + return { rule: OBJECT_LITERAL_STEP, what: "is an object literal" }; + } + if (ts.isNewExpression(node)) { + const type = unwrap(node.expression); + if (ts.isClassExpression(type)) { + return { rule: LOCAL_TYPE_STEP, what: "is an instance of a class with no name" }; + } + if (ts.isIdentifier(type)) { + if (type.text === "Pipeline" && role === "Pipeline") { + return { rule: UNNAMED_PIPELINE, what: "is a plain `new Pipeline()`" }; + } + const declaration = resolve(type); + if (declaration !== undefined && ts.isClassDeclaration(declaration) && isInsideFunction(declaration)) { + return { rule: LOCAL_TYPE_STEP, what: `is an instance of \`${type.text}\`, a class that is declared in a function` }; + } + } + return undefined; + } + if (ts.isIdentifier(node)) { + const declaration = resolve(node); + if (declaration === undefined) return undefined; + if (ts.isFunctionDeclaration(declaration)) { + return role === "Valve predicate" + ? { rule: FUNCTION_PREDICATE, what: `\`${node.text}\` is a function, not a predicate object` } + : { rule: NAMED_FUNCTION_STEP, what: `\`${node.text}\` is a function` }; + } + if (ts.isVariableDeclaration(declaration) && isConst(declaration) && declaration.initializer !== undefined) { + const verdict = classifyStep(declaration.initializer, role, seen); + return verdict === undefined ? undefined : { rule: verdict.rule, what: `\`${node.text}\` ${verdict.what}` }; + } + } + return undefined; +} diff --git a/packages/codeupipe-lint-ts/tests/anonymous-steps.test.ts b/packages/codeupipe-lint-ts/tests/anonymous-steps.test.ts new file mode 100644 index 0000000..1de5589 --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/anonymous-steps.test.ts @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { lintIn } from "./lint-in.ts"; + +const HEAD = `import { Pipeline } from "codeupipe"; +class AddFilter { call(payload) { return payload; } } +class IsOnPredicate { test(payload) { return true; } } +const pipeline = new Pipeline(); +`; + +test("named classes at module level are clean", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +pipeline.addFilter(new AddFilter()); +pipeline.addValve(new IsOnPredicate(), new AddFilter()); +const kept = new AddFilter(); +pipeline.addFilter(kept); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, []); + assert.equal(out.exitCode, 0); + assert.match(out.report, /^No findings\.$/); +}); + +test("CUP-A001: an arrow function or a function expression as a step", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +pipeline.addFilter((payload) => payload); +pipeline.addTap(function (payload) {}); +const later = (payload) => payload; +pipeline.addFilter(later); +pipeline.addValve((payload) => true, new AddFilter()); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:6 CUP-A001", "build.ts:7 CUP-A001", "build.ts:9 CUP-A001", "build.ts:10 CUP-A001"]); + assert.match(out.findings[0]?.message ?? "", /^Filter is a function\./); + assert.match(out.findings[2]?.message ?? "", /^Filter `later` is a function\./); + assert.match(out.findings[3]?.message ?? "", /^Valve predicate is a function\./); + assert.equal(out.exitCode, 1); +}); + +test("CUP-A002: a named function as a Filter or a Tap", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +function normalize(payload) { return payload; } +pipeline.addFilter(normalize); +pipeline.addTap(normalize); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:7 CUP-A002", "build.ts:8 CUP-A002"]); +}); + +test("CUP-A003: a class that is declared in a function, or that has no name", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +function make() { + class LocalFilter { call(payload) { return payload; } } + pipeline.addFilter(new LocalFilter()); + const kept = new LocalFilter(); + pipeline.addFilter(kept); +} +pipeline.addFilter(new (class { call(payload) { return payload; } })()); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:8 CUP-A003", "build.ts:10 CUP-A003", "build.ts:12 CUP-A003"]); +}); + +test("CUP-A005: a plain Pipeline as a step", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +class InnerPipeline extends Pipeline {} +pipeline.addPipeline(new Pipeline()); +const inner = new Pipeline(); +pipeline.addPipeline(inner); +pipeline.addPipeline(new InnerPipeline()); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:7 CUP-A005", "build.ts:9 CUP-A005"]); +}); + +test("CUP-A008: a named function as a Valve predicate", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +function isOn(payload) { return true; } +pipeline.addValve(isOn, new AddFilter()); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:7 CUP-A008"]); +}); + +test("CUP-A009: an object literal as a step", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +pipeline.addFilter({ call(payload) { return payload; } }); +const tap = { observe(payload) {} }; +pipeline.addTap(tap); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:6 CUP-A009", "build.ts:8 CUP-A009"]); +}); + +test("a name that the file does not bind, and a let, are not flagged", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +import { imported } from "./other.ts"; +let changed = (payload) => payload; +changed = new AddFilter(); +pipeline.addFilter(imported); +pipeline.addFilter(changed); +function use(parameter) { pipeline.addFilter(parameter); } +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, []); +}); + +test("this.addFilter in a pipeline class is checked too", async () => { + const out = await lintIn({ "pricing-pipeline.ts": `import { Pipeline } from "codeupipe"; +export class PricingPipeline extends Pipeline { + constructor() { + super(); + this.addFilter((payload) => payload); + } +} +` }); + assert.deepEqual(out.found, ["pricing-pipeline.ts:5 CUP-A001"]); +}); + +test("cup: allow-anonymous suppresses a finding on its line, the line above, or the line of the call", async () => { + const out = await lintIn({ "build.ts": HEAD + ` +pipeline.addFilter((payload) => payload); // cup: allow-anonymous a test of the refusal +// cup: allow-anonymous +pipeline.addFilter((payload) => payload); +pipeline.addValve( // cup: allow-anonymous + (payload) => true, + new AddFilter()); +pipeline.addFilter((payload) => payload); +` }, { disable: ["CUP-I005", "CUP-I006"] }); + assert.deepEqual(out.found, ["build.ts:12 CUP-A001"]); + assert.equal(out.suppressed, 3); + assert.match(out.report, /1 finding \(3 suppressed\)\.$/); +}); diff --git a/packages/codeupipe-lint-ts/tests/cli.test.ts b/packages/codeupipe-lint-ts/tests/cli.test.ts new file mode 100644 index 0000000..e561683 --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/cli.test.ts @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { lintIn } from "./lint-in.ts"; + +const CLI = join(import.meta.dirname, "../src/cli.ts"); +const BAD = `export class Discount { call(payload) { return payload; } }\n`; + +function project(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "cup-lint-cli-")); + for (const [name, text] of Object.entries(files)) writeFileSync(join(root, name), text); + return root; +} + +function run(cwd: string, ...args: string[]) { + const done = spawnSync(process.execPath, [CLI, ...args], { cwd, encoding: "utf8" }); + return { code: done.status, out: done.stdout, err: done.stderr }; +} + +test("exit 0 with no findings, exit 1 with findings, one line for each", () => { + const root = project({ "discount.ts": BAD, "set-filter.ts": `export class SetFilter { call(payload) { return payload; } }\n` }); + const clean = run(root, "set-filter.ts"); + assert.equal(clean.code, 0); + assert.equal(clean.out.trim(), "No findings."); + const bad = run(root, "."); + assert.equal(bad.code, 1); + assert.match(bad.out, /^discount\.ts:1:14 CUP-I006 primitive-file-name /m); + assert.match(bad.out, /^2 findings\.$/m); +}); + +test("exit 2 for a path that is not there, and for a rule that is not known", () => { + const root = project({}); + assert.equal(run(root, "nothing-here").code, 2); + const unknown = run(root, "--disable", "CUP-X999", "."); + assert.equal(unknown.code, 2); + assert.match(unknown.err, /no such rule: CUP-X999/); + assert.equal(run(root).code, 2); +}); + +test("--rules lists the rule ids", () => { + const listed = run(project({}), "--rules"); + assert.equal(listed.code, 0); + assert.match(listed.out, /^CUP-A001 lambda-step$/m); + assert.match(listed.out, /^CUP-I007 primitive-role-suffix$/m); +}); + +test("--exclude and --disable", () => { + const root = project({ "discount.ts": BAD }); + assert.equal(run(root, "--exclude", "*/discount.ts", "--exclude", "discount.ts", ".").code, 0); + assert.equal(run(root, "--disable", "CUP-I006", "--disable", "CUP-I007", ".").code, 0); + assert.equal(run(root, "--disable", "CUP-I006", ".").code, 1); +}); + +test("with no paths, they come from cup.config.json", () => { + const root = project({ + "discount.ts": BAD, + "cup.config.json": JSON.stringify({ lint: { paths: ["."], disable: ["CUP-I006", "CUP-I007"] } }), + }); + assert.equal(run(root).code, 0); + assert.equal(run(root, "discount.ts").code, 1); + writeFileSync(join(root, "cup.config.json"), JSON.stringify({ lint: { path: ["."] } })); + const wrong = run(root); + assert.equal(wrong.code, 2); + assert.match(wrong.err, /unknown key 'path'/); +}); + +test("node_modules and dist are not read", async () => { + const out = await lintIn({ "node_modules/x/discount.ts": BAD, "dist/discount.ts": BAD, "types.d.ts": BAD }); + assert.deepEqual(out.found, []); +}); + +test("the linter is clean on the TypeScript code of this repository", () => { + const here = join(import.meta.dirname, ".."); + const done = run(here, "src", "../codeupipe-ts/src", "../codeupipe-ts/examples", "../codeupipe-compile-ts/src", + "../../conformance/runners/typescript"); + // The plain logic of this repository is reported as warnings. A warning does not fail the run. + assert.match(done.out, /^No findings[ .]/m); + assert.equal(done.code, 0); +}); diff --git a/packages/codeupipe-lint-ts/tests/invariants.test.ts b/packages/codeupipe-lint-ts/tests/invariants.test.ts new file mode 100644 index 0000000..01a0c13 --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/invariants.test.ts @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { lintIn } from "./lint-in.ts"; + +test("CUP-I003: a Filter or a Predicate that reads State", async () => { + const out = await lintIn({ + "count-filter.ts": `import type { Payload, Pipeline, State } from "codeupipe"; +export class CountFilter { + pipeline: Pipeline; + seen: State; + state = 3; + constructor(pipeline: Pipeline, seen: State) { + this.pipeline = pipeline; + this.seen = seen; + } + call(payload: Payload): Payload { + payload.insert("a", this.pipeline.state.executed.length); + payload.insert("b", this.seen.skipped); + payload.insert("c", this.state); + payload.insert("d", payload.get("state")); + return payload; + } +} +`, + "ran-predicate.ts": `export class RanPredicate { + holder; + constructor(holder) { this.holder = holder; } + test(payload) { return this.holder.state.executed.length > 0; } +} +`, + "watch-tap.ts": `export class WatchTap { + pipeline; + observe(payload) { console.log(this.pipeline.state.executed); } +} +`, + }); + assert.deepEqual(out.found, ["count-filter.ts:11 CUP-I003", "count-filter.ts:12 CUP-I003", "ran-predicate.ts:4 CUP-I003"]); +}); + +test("CUP-I004: a write to the Payload from a Tap, a Predicate or a Hook", async () => { + const out = await lintIn({ + "mark-tap.ts": `export class MarkTap { + rows = []; + observe(payload) { + payload.insert("seen", true).remove("x"); + payload.copy().insert("mine", 1); + this.rows.insert(0, payload.get("a")); + } +} +`, + "is-new-predicate.ts": `export class IsNewPredicate { + test(p) { p.withTrace("t"); return true; } +} +`, + "stamp-hook.ts": `import { Hook } from "codeupipe"; +export class StampHook extends Hook { + override before(step, payload) { payload.merge(payload); } + override onError(step, error, payload) { payload.insert("failed", true); } +} +`, + "set-filter.ts": `export class SetFilter { + call(payload) { return payload.insert("a", 1); } +} +`, + }); + assert.deepEqual(out.found, ["is-new-predicate.ts:2 CUP-I004", "mark-tap.ts:4 CUP-I004", + "stamp-hook.ts:3 CUP-I004", "stamp-hook.ts:4 CUP-I004"]); + assert.match(out.findings[1]?.message ?? "", /^`observe` writes to the Payload \(`payload\.remove`\)\./); +}); + +test("CUP-I005: one primitive in each file", async () => { + const out = await lintIn({ + "set-filter.ts": `export class SetFilter { call(payload) { return payload; } } +class Helper {} +export class SeeTap { observe(payload) {} } +function make() { class LocalFilter { call(payload) { return payload; } } } +`, + "tests/steps.ts": `class AFilter { call(payload) { return payload; } } +class BFilter { call(payload) { return payload; } } +`, + "steps.test.ts": `class AFilter { call(payload) { return payload; } } +class BFilter { call(payload) { return payload; } } +`, + }); + assert.deepEqual(out.found, ["set-filter.ts:3 CUP-I005", "set-filter.ts:3 CUP-I006"]); + assert.match(out.findings[0]?.message ?? "", /^Tap `SeeTap` shares this file with Filter `SetFilter`\./); +}); + +test("CUP-I006: the file name is the class name in kebab-case", async () => { + const out = await lintIn({ + "discount.ts": `export class DiscountFilter { call(payload) { return payload; } }\n`, + "parse-html-filter.ts": `export class ParseHTMLFilter { call(payload) { return payload; } }\n`, + "pricing-pipeline.tsx": `import { Pipeline } from "codeupipe";\nexport class PricingPipeline extends Pipeline {}\n`, + }); + assert.deepEqual(out.found, ["discount.ts:1 CUP-I006"]); + assert.match(out.findings[0]?.message ?? "", /`DiscountFilter` is in `discount\.ts`\. Put it in `discount-filter\.ts`\./); +}); + +test("CUP-I007: the class name ends with its role", async () => { + const out = await lintIn({ + "discount.ts": `export class Discount { call(payload) { return payload; } }\n`, + "audit.ts": `import type { Tap } from "codeupipe";\nexport class Audit implements Tap { observe() {} }\n`, + "at-least.ts": `export class AtLeast { test(payload) { return true; } }\n`, + "print-steps.ts": `import { Hook } from "codeupipe";\nexport class PrintSteps extends Hook {}\n`, + "pricing.ts": `import { Pipeline } from "codeupipe";\nexport class Pricing extends Pipeline {}\n`, + "point.ts": `export class Point { call(a, b) {} test() {} observe(options = {}) {} }\n`, + }); + assert.deepEqual(out.found, ["at-least.ts:1 CUP-I006", "at-least.ts:1 CUP-I007", "audit.ts:2 CUP-I006", "audit.ts:2 CUP-I007", + "discount.ts:1 CUP-I006", "discount.ts:1 CUP-I007", "pricing.ts:2 CUP-I006", "pricing.ts:2 CUP-I007", + "print-steps.ts:2 CUP-I006", "print-steps.ts:2 CUP-I007"]); + assert.match(out.findings[1]?.message ?? "", /^Predicate `AtLeast` does not say what it is\. Name it `AtLeastPredicate`\./); +}); + +test("the comment markers suppress the invariant findings", async () => { + const out = await lintIn({ + "discount.ts": `// cup: allow-name\nexport class Discount { call(payload) { return payload; } }\n`, + "mark-tap.ts": `export class MarkTap {\n observe(payload) {\n payload.insert("a", 1); // cup: allow-outside-write\n }\n}\n`, + }); + assert.deepEqual(out.found, []); + assert.equal(out.suppressed, 3); +}); + +test("a file with a syntax error is CUP-I000", async () => { + const out = await lintIn({ "broken.ts": `export class {{{\n` }); + assert.deepEqual(out.found.map((line) => line.split(" ")[1]), ["CUP-I000"]); +}); diff --git a/packages/codeupipe-lint-ts/tests/lint-in.ts b/packages/codeupipe-lint-ts/tests/lint-in.ts new file mode 100644 index 0000000..5a1feed --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/lint-in.ts @@ -0,0 +1,28 @@ +// A helper for the tests: write source text to a new directory and lint it. +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; + +import { lintPaths } from "../src/index.ts"; +import type { Finding } from "../src/index.ts"; + +export type Linted = { found: string[]; findings: Finding[]; suppressed: number; report: string; exitCode: number }; + +/** `found` holds one string for each finding: `file:line rule`. */ +export async function lintIn(files: Record, + options: { exclude?: string[]; disable?: string[]; strict?: boolean } = {}): Promise { + const root = mkdtempSync(join(tmpdir(), "cup-lint-")); + for (const [name, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, name)), { recursive: true }); + writeFileSync(join(root, name), text); + } + const out = await lintPaths([root], options); + const findings = out.get("findings") ?? []; + return { + findings, + found: findings.map((f) => `${relative(root, f.path)}:${f.line} ${f.rule}`), + suppressed: (out.get("suppressed") ?? []).length, + report: out.get("report") ?? "", + exitCode: out.get("exitCode") ?? -1, + }; +} diff --git a/packages/codeupipe-lint-ts/tests/plain-logic.test.ts b/packages/codeupipe-lint-ts/tests/plain-logic.test.ts new file mode 100644 index 0000000..a41392f --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/plain-logic.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { lintIn } from "./lint-in.ts"; + +const warnings = (out: Awaited>) => + out.findings.filter((finding) => finding.severity === "warning").map((finding) => finding.message.split(" has logic")[0]); + +test("CUP-I008: a function with a decision that is not in a primitive is a warning", async () => { + const out = await lintIn({ + "who-is-due.ts": `export function whoIsDue(entries) { if (entries.length === 0) return "say"; return "model"; } +export const pick = (list) => { for (const each of list) { if (each) return each; } return null; }; +export const choose = (flag) => (flag ? "a" : "b"); +export async function load(path) { try { return await read(path); } catch { return null; } } +export function plain(a, b) { return a + b; } +export const double = (n) => n * 2; +`, + }); + assert.deepEqual(out.found, ["who-is-due.ts:1 CUP-I008", "who-is-due.ts:2 CUP-I008", "who-is-due.ts:3 CUP-I008", "who-is-due.ts:4 CUP-I008"]); + assert.deepEqual(warnings(out), ["Function `whoIsDue`", "Function `pick`", "Function `choose`", "Function `load`"]); + assert.match(out.findings[0]?.message ?? "", /is not a Filter, a Predicate, a Tap, a Hook or a Pipeline\. Write it as a primitive/); +}); + +test("CUP-I008: a class that is not a primitive, and code at the top of a file", async () => { + const out = await lintIn({ + "record-store.ts": `export class RecordStore { + list() { return this.files.filter((file) => file.endsWith(".jsonl")); } + read(id) { if (!this.has(id)) throw new Error("none"); return []; } +} +export class Point { constructor(x) { this.x = x; } get double() { return this.x * 2; } } +`, + "main.ts": `import { run } from "./run.ts"; +const args = process.argv.slice(2); +if (args.length === 0) process.exit(2); +for (const arg of args) run(arg); +`, + }); + assert.deepEqual(out.found, ["main.ts:3 CUP-I008", "record-store.ts:1 CUP-I008"]); + assert.deepEqual(warnings(out), ["The code at the top of this file", "Class `RecordStore`"]); +}); + +test("CUP-I008: logic in a primitive is where it must be, and types have no logic", async () => { + const out = await lintIn({ + "pick-filter.ts": `export class PickFilter { call(payload) { if (payload.has("a")) return payload.insert("b", 1); for (const k of []) {} return payload; } }\n`, + "is-on-predicate.ts": `export class IsOnPredicate { test(payload) { return payload.get("on") ? true : false; } }\n`, + "see-tap.ts": `export class SeeTap { observe(payload) { if (payload.has("x")) console.log(1); } }\n`, + "log-hook.ts": `import { Hook } from "codeupipe";\nexport class LogHook extends Hook { override before(step) { if (step) console.log(step); } }\n`, + "main-pipeline.ts": `import { Pipeline } from "codeupipe";\nexport class MainPipeline extends Pipeline { constructor() { super(); if (process.env.X) this.observe(); } }\n`, + "types.ts": `export type Due = "say" | "model";\nexport interface Model { reply(text: string): string }\nexport const LIMIT = 20000;\nexport const NAMES = ["a", "b"];\n`, + }); + assert.deepEqual(out.found, []); +}); + +test("a warning does not fail the run, unless the run is strict", async () => { + const files = { "helper.ts": `export function pick(a) { return a ? 1 : 2; }\n` }; + const soft = await lintIn(files); + assert.equal(soft.exitCode, 0); + assert.match(soft.report, /helper\.ts:1:17 CUP-I008 logic-outside-primitive warning: Function `pick` has logic/); + assert.match(soft.report, /^No findings\. 1 warning\.$/m); + const strict = await lintIn(files, { strict: true }); + assert.equal(strict.exitCode, 1); +}); + +test("a warning and a finding are counted apart", async () => { + const out = await lintIn({ "discount.ts": `export class Discount { call(payload) { return payload; } }\nexport function pick(a) { return a ? 1 : 2; }\n` }); + assert.equal(out.exitCode, 1); + assert.match(out.report, /^2 findings\. 1 warning\.$/m); +}); + +test("cup: allow-plain with a reason suppresses the warning, and test files are not checked", async () => { + const out = await lintIn({ + "helper.ts": `// cup: allow-plain a pure helper of two Predicates\nexport function pick(a) { return a ? 1 : 2; }\n`, + "helper.test.ts": `export function pick(a) { return a ? 1 : 2; }\n`, + "tests/support.ts": `export function pick(a) { return a ? 1 : 2; }\n`, + }); + assert.deepEqual(out.found, []); + assert.equal(out.suppressed, 1); + assert.match(out.report, /^No findings \(1 suppressed\)\.$/m); +}); + +test("the rule can be switched off", async () => { + const out = await lintIn({ "helper.ts": `export function pick(a) { return a ? 1 : 2; }\n` }, { disable: ["CUP-I008"] }); + assert.deepEqual(out.found, []); + assert.match(out.report, /^No findings\.$/m); +}); diff --git a/packages/codeupipe-lint-ts/tests/stale-markers.test.ts b/packages/codeupipe-lint-ts/tests/stale-markers.test.ts new file mode 100644 index 0000000..4a329d4 --- /dev/null +++ b/packages/codeupipe-lint-ts/tests/stale-markers.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { lintIn } from "./lint-in.ts"; + +test("CUP-I009: a comment that allows nothing is a warning", async () => { + const out = await lintIn({ + "add.ts": `// cup: allow-plain it had a loop before +export function add(a, b) { return a + b; } +export const one = 1; // cup: allow-state-read nothing here reads State +/* + * cup: allow-plian a kind that does not exist + */ +export const two = 2; +`, + }); + assert.deepEqual(out.found, ["add.ts:1 CUP-I009", "add.ts:3 CUP-I009", "add.ts:5 CUP-I009"]); + assert.equal(out.findings[0]?.severity, "warning"); + assert.match(out.findings[0]?.message ?? "", /^The comment `cup: allow-plain` allows nothing\. Remove it/); + assert.equal(out.exitCode, 0); + assert.equal((await lintIn({ "add.ts": "// cup: allow-plain no reason\nexport const one = 1;\n" }, { strict: true })).exitCode, 1); +}); + +test("CUP-I009: a comment that allows a finding is not reported, and text that only names a marker is not a comment", async () => { + const out = await lintIn({ + "pick.ts": `// cup: allow-plain the entry of the program +export function pick(list) { for (const each of list) { if (each) return each; } return null; } +export function choose(flag) { return flag ? "a" : "b"; } // cup: allow-plain a small helper +/** The marker \`// cup: allow-plain\` allows a finding. */ +export const HELP = "say why: // cup: allow-plain "; +export const MORE = \`\${HELP} // cup: allow-plain\`; +`, + }); + assert.deepEqual(out.found, []); + assert.equal(out.suppressed, 2); +}); + +test("CUP-I009: a comment for a rule that is switched off is not reported, and the rule can be switched off", async () => { + const files = { "pick.ts": "// cup: allow-plain the entry\nexport function pick(flag) { return flag ? 1 : 2; }\n" }; + const off = await lintIn(files, { disable: ["CUP-I008"] }); + assert.deepEqual(off.found, []); + assert.equal(off.suppressed, 0); + assert.deepEqual((await lintIn({ "one.ts": "// cup: allow-plain no reason\nexport const one = 1;\n" }, { disable: ["CUP-I009"] })).found, []); +}); diff --git a/packages/codeupipe-lint-ts/tsconfig.build.json b/packages/codeupipe-lint-ts/tsconfig.build.json new file mode 100644 index 0000000..7f9bdff --- /dev/null +++ b/packages/codeupipe-lint-ts/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src", + "outDir": "dist", + "declaration": true + }, + "include": ["src"] +} diff --git a/packages/codeupipe-lint-ts/tsconfig.json b/packages/codeupipe-lint-ts/tsconfig.json new file mode 100644 index 0000000..d05c8e5 --- /dev/null +++ b/packages/codeupipe-lint-ts/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src", "tests"] +} diff --git a/packages/codeupipe-lint/README.md b/packages/codeupipe-lint/README.md index a3660e4..8f2a807 100644 --- a/packages/codeupipe-lint/README.md +++ b/packages/codeupipe-lint/README.md @@ -13,7 +13,7 @@ Each rule can be switched off. There are three ways. | For | How | |---|---| -| One line | A comment on the line: `# cup: allow-anonymous`, `# cup: allow-mutation`, `# cup: allow-state-read`, `# cup: allow-outside-write`, `# cup: allow-shared-file`, `# cup: allow-name` | +| One line | A comment on the line: `# cup: allow-anonymous`, `# cup: allow-mutation`, `# cup: allow-state-read`, `# cup: allow-outside-write`, `# cup: allow-shared-file`, `# cup: allow-name`, `# cup: allow-plain`. A comment that allows no finding is itself a finding (CUP-I009) | | Some files | `--exclude GLOB`, or `lint.exclude` in `cup.config.json` | | The whole project | `--disable RULE`, or `lint.disable` in `cup.config.json` | @@ -26,10 +26,10 @@ in the list: each linter uses the ids that are its own. ## Anonymous-step linter -Step identity (conformance/API.md, "Step identity") makes a step's identity a -pointer to its filter. Anonymous steps have no such pointer and are -unsupported. The runtime refuses them when the step is added; this linter -flags them at authoring time. +A step is reported by the name of its class (conformance/API.md, "Step +labels"). A step that is a lambda, a function, a class defined inside a +function, or a plain nested `Pipeline` has no useful name and cannot be +imported and tested alone. This linter flags them when the code is written. ```bash python -m codeupipe_lint.anonymous_steps # files or directories @@ -40,7 +40,7 @@ python -m codeupipe_lint.anonymous_steps --rules # list the rule i Each finding is one line, and the message names the fix: ``` -path:line:col CUP-A001 lambda-step Valve predicate is a lambda. Define it at module level (...) so it has a stable identity and can be tested on its own. +path:line:col CUP-A001 lambda-step Valve predicate is a lambda. Define it at module level (...) so it has a name in State and can be tested on its own. ``` With no paths, both linters read them from `cup.config.json` @@ -61,7 +61,6 @@ returns the result Payload (`findings`, `suppressed`, `errors`, `report`, `exit_ | CUP-A002 | local-function-step | a named function declared inside a function | | CUP-A003 | local-type-step | a class declared inside a function, or an instance of one | | CUP-A005 | unnamed-pipeline-step | a plain, unnamed `Pipeline` nested as a step; nested pipelines must be named Pipeline types | -| CUP-A006 | entry-script-step | a class or function of a script (`__main__`), added from its `if __name__ == "__main__":` block | | CUP-A008 | function-predicate | a named function as a Valve predicate. Predicates are objects with `test(payload)` (conformance/API.md, "Valve") | A step is any argument of `add_filter`, `add_tap` or `add_pipeline` (on any @@ -75,19 +74,15 @@ A comment containing `# cup: allow-anonymous` (any text may follow) suppresses a finding when it is on the finding's line, on the line of the enclosing add or Valve call, or alone on the line directly above. Suppressed findings are counted in the summary. Use it only where the anonymous step is -the point, as in the tests that exercise the runtime's refusal -(`tests/core/test_step_identity.py`). +the point, as in a test of what the runtime does with one +(`tests/core/test_step_labels.py`). ### Limits Names resolve lexically (innermost function scope out to the module; a parameter, loop target or import shadows). A name bound two different ways -in one scope is not flagged. CUP-A006 is deliberately conservative: only a -module with a `__main__` guard, only steps added from that block or from a -module-level function it calls (transitively); a script without a guard is -not flagged (the runtime still refuses it). Not seen: `functools.partial`, -`type(...)`-built classes, steps built in another module (all refused at -runtime). +in one scope is not flagged. Not seen: `functools.partial`, +`type(...)`-built classes, steps built in another module. ### Layout @@ -129,6 +124,8 @@ a path could not be read. From Python: | CUP-I005 | one-primitive-per-file | a second Filter, Predicate, Tap, Hook or named Pipeline class at module level in one file | | CUP-I006 | primitive-file-name | the file of a primitive is not its class name in snake_case (`DiscountFilter` is in `discount_filter.py`) | | CUP-I007 | primitive-role-suffix | the class name of a primitive does not end with its role: `Filter`, `Tap`, `Hook`, `Predicate` or `Pipeline` | +| CUP-I008 | logic-outside-primitive | **a warning.** A function, a class or the code at module level has logic (`if`, `a if c else b`, `match`, a loop, `try`) and is not a primitive | +| CUP-I009 | marker-allows-nothing | **a warning.** A `# cup: allow-...` comment that allowed no finding: the code that it was for is gone or has moved, or its kind has a spelling error | **Why CUP-I002.** A Payload owns its keys, not its values. `payload.get("items")` returns the live list, shared with every copy of the @@ -164,10 +161,30 @@ the standard style for a Python module, so one name gives the other. A subclass of `Valve` can end with `Valve`. Test files are not checked. Suppress with `cup: allow-name`. +**Why CUP-I008.** A project that uses codeupipe writes its logic as Filters, +Predicates, Taps, Hooks and Pipelines. Logic in a plain function or a plain +class works as well, so nothing else shows the mistake. Flagged, once each: +a function at module level, a class at module level that is not a primitive, +and the code at module level. Not flagged: a primitive, a function or a class +with no logic, a test file, and the guards `if __name__ == "__main__":` and +`if TYPE_CHECKING:`. Not seen: a comprehension, and `and` / `or`. For logic +that must be plain code (the edge to another program, the entry of a +program), say why: `# cup: allow-plain `. + +**Warnings.** CUP-I008 and CUP-I009 are warnings. A warning is in the report +and is counted apart (`No invariant violations. 3 warnings.`). It does not +make the exit code 1. With `--strict`, a warning fails the run as a finding +does. + +**Why CUP-I009.** A comment that allows nothing says that the code has an +exception that it does not have. Only a comment that starts with the marker +counts. A `cup: allow-anonymous` comment is for the anonymous-step linter and +is not looked at. + **Why CUP-I003.** A Filter's result and a Valve's decision must depend only on the Payload. A step is never handed State, so the only way to read it is to hold a pipeline or a State, and that makes the step's behaviour depend on -what ran before it, which the static export cannot reproduce. Taps and Hooks +what ran before it, which the compiled form cannot reproduce. Taps and Hooks only watch, so `observe` and Hook methods are not checked. `self.state` is not flagged when it is the step's own attribute. `.state` diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/__main__.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/__main__.py index 956f743..28330ec 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/__main__.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/__main__.py @@ -16,6 +16,7 @@ from ..rule_ids import unknown_rules +# cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result def main(argv=None) -> int: parser = argparse.ArgumentParser(prog="python -m codeupipe_lint.anonymous_steps", description="Flag anonymous pipeline steps (packages/codeupipe-lint/README.md).") diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/apply_suppressions_filter.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/apply_suppressions_filter.py index 860f026..bfa4d17 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/apply_suppressions_filter.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/apply_suppressions_filter.py @@ -17,7 +17,8 @@ class ApplySuppressionsFilter: """Filter (sync). Input keys: ``findings``. - Output keys: ``findings`` (kept), ``suppressed`` (dropped). + Output keys: ``findings`` (kept), ``suppressed`` (dropped), ``used_markers`` + (the place of each comment that allowed a finding, as ``(path, line)``). ``marker`` is the comment that suppresses; other linters pass their own. """ @@ -26,7 +27,7 @@ def __init__(self, marker=MARKER): def call(self, payload: Payload) -> Payload: cache = {} - kept, suppressed = [], [] + kept, suppressed, used = [], [], set() for finding in payload.get("findings", []): lines = cache.get(finding.path) if lines is None: @@ -36,17 +37,25 @@ def call(self, payload: Payload) -> Payload: except OSError: lines = [] cache[finding.path] = lines - (suppressed if _is_suppressed(finding, lines, self.marker) else kept).append(finding) - return payload.insert("findings", kept).insert("suppressed", suppressed) - - -def _line(lines, number): - return lines[number - 1] if number and 0 < number <= len(lines) else "" + places = ApplySuppressionsFilter._marker_lines(finding, lines, self.marker) + used.update((finding.path, place) for place in places) + (suppressed if places else kept).append(finding) + return (payload.insert("findings", kept).insert("suppressed", suppressed) + .insert("used_markers", sorted(used))) + + @staticmethod + def _line(lines, number): + return lines[number - 1] if number and 0 < number <= len(lines) else "" + + @staticmethod + def _marker_lines(finding, lines, marker=MARKER): + """The lines of the comments that allow this finding. Empty when none does.""" + places = [number for number in (finding.line, finding.call_line) if marker.search(ApplySuppressionsFilter._line(lines, number))] + above = ApplySuppressionsFilter._line(lines, finding.line - 1) + if _COMMENT_ONLY.match(above) and marker.search(above): + places.append(finding.line - 1) + return places def _is_suppressed(finding, lines, marker=MARKER) -> bool: - for number in (finding.line, finding.call_line): - if marker.search(_line(lines, number)): - return True - above = _line(lines, finding.line - 1) - return bool(_COMMENT_ONLY.match(above) and marker.search(above)) + return bool(ApplySuppressionsFilter._marker_lines(finding, lines, marker)) diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/drop_disabled_filter.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/drop_disabled_filter.py index 60390c7..906a400 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/drop_disabled_filter.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/drop_disabled_filter.py @@ -6,13 +6,17 @@ class DropDisabledFilter: """Filter (sync). - Input keys: ``findings``, optional ``disable`` (rule ids). - Output keys: ``findings`` without the findings of a disabled rule. A - disabled rule is not counted as suppressed: it was not asked for. + Input keys: ``findings``, optional ``suppressed``, optional ``disable`` (rule ids). + Output keys: ``findings`` and ``suppressed`` without the findings of a + disabled rule. A disabled rule is not counted as suppressed: it was not + asked for. """ def call(self, payload: Payload) -> Payload: disabled = set(payload.get("disable") or ()) if not disabled: return payload - return payload.insert("findings", [f for f in payload.get("findings", []) if f.rule not in disabled]) + payload.insert("findings", [f for f in payload.get("findings", []) if f.rule not in disabled]) + if payload.has("suppressed"): + payload.insert("suppressed", [f for f in payload.get("suppressed", []) if f.rule not in disabled]) + return payload diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/finding.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/finding.py index cfb9f1c..6f0d32d 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/finding.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/finding.py @@ -3,11 +3,17 @@ from typing import NamedTuple, Optional +# What a report line has in front of the message, for each severity. +_PREFIX = {"warning": "warning: "} + + class Finding(NamedTuple): """An anonymous step at ``path:line:col`` (1-based line and column). ``call_line`` is the line of the enclosing ``add_*`` / Valve call, when it differs from ``line``: a suppression comment may sit on either. + ``severity`` is ``"warning"`` for a finding that is reported and does not + fail the run, unless the run is strict. """ path: str @@ -18,6 +24,8 @@ class Finding(NamedTuple): message: str lang: str call_line: Optional[int] = None + severity: Optional[str] = None def format(self) -> str: - return "%s:%d:%d %s %s %s" % (self.path, self.line, self.col, self.rule, self.slug, self.message) + prefix = _PREFIX.get(self.severity, "") + return "%s:%d:%d %s %s %s%s" % (self.path, self.line, self.col, self.rule, self.slug, prefix, self.message) diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/format_report_filter.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/format_report_filter.py index dd8a49f..d8c6c20 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/format_report_filter.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/format_report_filter.py @@ -6,11 +6,14 @@ class FormatReportFilter: """Filter (sync). - Input keys: ``findings``, ``suppressed``, ``errors``. + Input keys: ``findings``, ``suppressed``, ``errors``, optional ``strict``. Output keys: ``findings`` (sorted), ``report`` (stdout text), ``error_report`` (stderr text), ``exit_code``: 0 clean, 1 anonymous steps found, 2 a path could not be read (the check is incomplete). ``noun`` and ``plural`` name what a finding is; other linters pass their own. + + A warning is in the report and is counted apart. It does not make the + exit code 1, unless ``strict`` is true. """ def __init__(self, noun="anonymous step", plural="anonymous steps"): @@ -22,14 +25,20 @@ def call(self, payload: Payload) -> Payload: suppressed = payload.get("suppressed", []) errors = payload.get("errors", []) lines = [f.format() for f in findings] - if findings: - summary = "%d %s" % (len(findings), self.noun if len(findings) == 1 else self.plural) + warnings = sum(1 for f in findings if f.severity == "warning") + real = len(findings) - warnings + if real: + summary = "%d %s" % (real, self.noun if real == 1 else self.plural) else: summary = "No %s" % self.plural if suppressed: summary += " (%d suppressed)" % len(suppressed) - lines.append(summary + ".") - exit_code = 2 if errors else (1 if findings else 0) + summary += "." + if warnings: + summary += " %d %s." % (warnings, "warning" if warnings == 1 else "warnings") + lines.append(summary) + fails = real or (warnings and payload.get("strict") is True) + exit_code = 2 if errors else (1 if fails else 0) error_report = "\n".join("error: " + e for e in errors) return (payload.insert("findings", findings).insert("report", "\n".join(lines)) .insert("error_report", error_report).insert("exit_code", exit_code)) diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/lint_pipeline.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/lint_pipeline.py index ae441ac..f8e0b7d 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/lint_pipeline.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/lint_pipeline.py @@ -34,8 +34,6 @@ def lint_paths(paths: Iterable[str], exclude: Optional[Iterable[str]] = None, """Run the linter over ``paths``; the result Payload holds ``findings``, ``suppressed``, ``errors``, ``report``, ``error_report`` and ``exit_code``.""" data = {"paths": list(paths), "findings": [], "errors": []} - if exclude: - data["exclude"] = list(exclude) - if disable: - data["disable"] = list(disable) + data["exclude"] = list(exclude or ()) + data["disable"] = list(disable or ()) return build_lint_pipeline().run(Payload(data)) diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/python_step_visitor.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/python_step_visitor.py index 2a94505..40fee67 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/python_step_visitor.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/python_step_visitor.py @@ -12,10 +12,7 @@ - as a Valve predicate, a named function (CUP-A008): predicates are objects with ``test(payload)``; - the result of calling a function in this module that returns one of the - above (a closure factory), reported with the same rule; -- a module-level class or function of a script, added from its - ``if __name__ == "__main__":`` block or a function that block calls - (CUP-A006). Only then: a module that is merely importable is fine. + above (a closure factory), reported with the same rule. Names are resolved lexically: innermost function scope outwards to the module, stopping at the first scope that binds the name. A name bound more @@ -28,7 +25,7 @@ from typing import Dict, List, Optional, Set, Tuple from .finding import Finding -from .rules import (ENTRY_SCRIPT, FUNCTION_PREDICATE, LAMBDA, LOCAL_FUNCTION, LOCAL_TYPE, UNNAMED_PIPELINE, +from .rules import (FUNCTION_PREDICATE, LAMBDA, LOCAL_FUNCTION, LOCAL_TYPE, UNNAMED_PIPELINE, make_finding) _STEP_METHODS = { @@ -48,51 +45,7 @@ _FUNCS = (ast.FunctionDef, ast.AsyncFunctionDef) -def _is_plain_pipeline_call(node: ast.AST) -> bool: - if not isinstance(node, ast.Call) or node.args or node.keywords: - return False - func = node.func - return (isinstance(func, ast.Name) and func.id == "Pipeline") or ( - isinstance(func, ast.Attribute) and func.attr == "Pipeline") - - -def _is_main_guard(node: ast.AST) -> bool: - if not isinstance(node, ast.If) or not isinstance(node.test, ast.Compare): - return False - test = node.test - if len(test.ops) != 1 or not isinstance(test.ops[0], ast.Eq): - return False - sides = [test.left, test.comparators[0]] - names = [s for s in sides if isinstance(s, ast.Name) and s.id == "__name__"] - consts = [s for s in sides if isinstance(s, ast.Constant) and s.value == "__main__"] - return bool(names) and bool(consts) - - -def _own_nodes(body: List[ast.stmt]): - """Every node in ``body``, not descending into nested functions, classes or lambdas.""" - stack = list(reversed(body)) - while stack: - node = stack.pop() - yield node - for child in reversed(list(ast.iter_child_nodes(node))): - if isinstance(child, _FUNCS + (ast.ClassDef, ast.Lambda)): - yield child # the definition itself binds a name; its body is its own scope - continue - stack.append(child) - - -def _target_names(target: ast.AST): - if isinstance(target, ast.Name): - yield target.id - elif isinstance(target, (ast.Tuple, ast.List)): - for element in target.elts: - for name in _target_names(element): - yield name - elif isinstance(target, ast.Starred): - for name in _target_names(target.value): - yield name - - +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code class PythonStepVisitor: """Collects Findings for one module (see the module docstring).""" @@ -103,13 +56,6 @@ def __init__(self, path: str, tree: ast.Module): self._findings: List[Finding] = [] self._seen: Set[Tuple[int, int, str]] = set() self._module_scope = self._scope(tree.body, None, is_module=True) - self._main_functions: Set[str] = set() - self._main_block: Optional[ast.If] = None - for stmt in tree.body: - if _is_main_guard(stmt): - self._main_block = stmt - if self._main_block is not None: - self._main_functions = self._reached_from_main() # -- scopes --------------------------------------------------------- @@ -126,7 +72,7 @@ def bind(name: str, binding: Binding) -> None: for arg in (args.vararg, args.kwarg): if arg is not None: bind(arg.arg, _OTHER) - nodes = list(_own_nodes(body)) + nodes = list(PythonStepVisitor._own_nodes(body)) local_classes = set() for node in nodes: if isinstance(node, _FUNCS): @@ -140,16 +86,16 @@ def bind(name: str, binding: Binding) -> None: targets = node.targets if isinstance(node, ast.Assign) else [node.target] binding = self._value_binding(node.value, local_classes, owner) for target in targets: - names = list(_target_names(target)) + names = list(PythonStepVisitor._target_names(target)) for name in names: bind(name, binding if isinstance(target, ast.Name) else _OTHER) elif isinstance(node, (ast.AugAssign, ast.For, ast.AsyncFor, ast.comprehension, ast.NamedExpr)): - for name in _target_names(node.target): + for name in PythonStepVisitor._target_names(node.target): bind(name, _OTHER) elif isinstance(node, (ast.With, ast.AsyncWith)): for item in node.items: if item.optional_vars is not None: - for name in _target_names(item.optional_vars): + for name in PythonStepVisitor._target_names(item.optional_vars): bind(name, _OTHER) elif isinstance(node, (ast.Import, ast.ImportFrom)): for alias in node.names: @@ -170,7 +116,7 @@ def bind(name: str, binding: Binding) -> None: def _value_binding(value: Optional[ast.AST], local_classes: Set[str], owner: Optional[str]) -> Binding: if isinstance(value, ast.Lambda): return ("lambda", "", owner, value) - if _is_plain_pipeline_call(value): + if PythonStepVisitor._is_plain_pipeline_call(value): return ("plain_pipeline", "", owner, value) if isinstance(value, ast.Call) and isinstance(value.func, ast.Name) and value.func.id in local_classes: return ("local_instance", value.func.id, owner, value) @@ -191,42 +137,26 @@ def _resolve(name: str, chain: List[Dict[str, Binding]]) -> Binding: return scope[name] return _OTHER - # -- __main__ ------------------------------------------------------- - - def _reached_from_main(self) -> Set[str]: - defs = {s.name: s for s in self._tree.body if isinstance(s, _FUNCS)} - reached: Set[str] = set() - frontier = [self._main_block] - while frontier: - node = frontier.pop() - for sub in ast.walk(node): - if isinstance(sub, ast.Call) and isinstance(sub.func, ast.Name): - name = sub.func.id - if name in defs and name not in reached: - reached.add(name) - frontier.append(defs[name]) - return reached - # -- classification ------------------------------------------------- - def _classify(self, expr: ast.AST, chain: List[Dict[str, Binding]], in_main: bool, + def _classify(self, expr: ast.AST, chain: List[Dict[str, Binding]], visiting: Set[int]) -> Optional[Tuple[str, str]]: """(rule, what) when ``expr`` is an anonymous step, else None. ``what`` starts with a verb.""" - got = self._describe(expr, chain, in_main, visiting) + got = self._describe(expr, chain, visiting) if got is None: return None rule, subject, noun = got return rule, ("`%s` is %s" % (subject, noun)) if subject else ("is " + noun) - def _describe(self, expr: ast.AST, chain: List[Dict[str, Binding]], in_main: bool, + def _describe(self, expr: ast.AST, chain: List[Dict[str, Binding]], visiting: Set[int]) -> Optional[Tuple[str, Optional[str], str]]: """(rule, subject name or None, noun phrase).""" if isinstance(expr, ast.Lambda): return LAMBDA, None, "a lambda" if isinstance(expr, ast.Name): - got = self._describe_binding(self._resolve(expr.id, chain), in_main) + got = self._describe_binding(self._resolve(expr.id, chain)) return None if got is None else (got[0], expr.id, got[1]) - if _is_plain_pipeline_call(expr): + if PythonStepVisitor._is_plain_pipeline_call(expr): return UNNAMED_PIPELINE, None, "a plain Pipeline()" if isinstance(expr, ast.Call) and isinstance(expr.func, ast.Name): name = expr.func.id @@ -234,8 +164,6 @@ def _describe(self, expr: ast.AST, chain: List[Dict[str, Binding]], in_main: boo kind = binding[0] if kind == "local_class": return LOCAL_TYPE, None, "an instance of %s, a class defined inside %s()" % (name, binding[2]) - if kind == "module_class" and in_main: - return ENTRY_SCRIPT, None, "an instance of %s, defined in the entry script" % name if kind in ("local_def", "module_def"): parent = chain if kind == "local_def" else [self._module_scope] returned = self._returned(binding[3], parent, visiting) @@ -246,7 +174,7 @@ def _describe(self, expr: ast.AST, chain: List[Dict[str, Binding]], in_main: boo return None @staticmethod - def _describe_binding(binding: Binding, in_main: bool) -> Optional[Tuple[str, str]]: + def _describe_binding(binding: Binding) -> Optional[Tuple[str, str]]: kind, name, owner = binding[0], binding[1], binding[2] if kind == "lambda": return LAMBDA, "a lambda" @@ -258,10 +186,6 @@ def _describe_binding(binding: Binding, in_main: bool) -> Optional[Tuple[str, st return LOCAL_TYPE, "an instance of %s, a class defined inside %s()" % (name, owner) if kind == "plain_pipeline": return UNNAMED_PIPELINE, "a plain Pipeline()" - if kind == "module_def" and in_main: - return ENTRY_SCRIPT, "a function defined in the entry script" - if kind == "module_class" and in_main: - return ENTRY_SCRIPT, "a class defined in the entry script" return None def _returned(self, func: ast.AST, parent_chain: List[Dict[str, Binding]], @@ -271,9 +195,9 @@ def _returned(self, func: ast.AST, parent_chain: List[Dict[str, Binding]], return None visiting = visiting | {id(func)} chain = self._chain_for(func, parent_chain) - for node in _own_nodes(func.body): + for node in PythonStepVisitor._own_nodes(func.body): if isinstance(node, ast.Return) and node.value is not None: - got = self._describe(node.value, chain, False, visiting) + got = self._describe(node.value, chain, visiting) if got is not None: return got return None @@ -281,41 +205,35 @@ def _returned(self, func: ast.AST, parent_chain: List[Dict[str, Binding]], # -- walk ----------------------------------------------------------- def findings(self) -> List[Finding]: - self._walk(self._tree, [self._module_scope], False) + self._walk(self._tree, [self._module_scope]) return self._findings - def _walk(self, node: ast.AST, chain: List[Dict[str, Binding]], in_main: bool) -> None: + def _walk(self, node: ast.AST, chain: List[Dict[str, Binding]]) -> None: for child in ast.iter_child_nodes(node): - child_main = in_main - if child is self._main_block: - child_main = True if isinstance(child, _FUNCS): - if chain[0] is self._module_scope and child.name in self._main_functions: - child_main = True - self._walk(child, self._chain_for(child, chain), child_main) + self._walk(child, self._chain_for(child, chain)) continue if isinstance(child, ast.Call): - self._check_call(child, chain, child_main) - self._walk(child, chain, child_main) + self._check_call(child, chain) + self._walk(child, chain) - def _check_call(self, call: ast.Call, chain: List[Dict[str, Binding]], in_main: bool) -> None: + def _check_call(self, call: ast.Call, chain: List[Dict[str, Binding]]) -> None: func = call.func if isinstance(func, ast.Attribute) and func.attr in _STEP_METHODS: role = _STEP_METHODS[func.attr] for arg in list(call.args) + [k.value for k in call.keywords]: elements = arg.elts if isinstance(arg, (ast.List, ast.Tuple, ast.Set)) else [arg] for element in elements: - self._report(element, role, call, chain, in_main) + self._report(element, role, call, chain) elif isinstance(func, ast.Attribute) and func.attr == "add_valve": for index, arg in enumerate(call.args): - self._report(arg, _VALVE_ROLES[min(index, 1)], call, chain, in_main) + self._report(arg, _VALVE_ROLES[min(index, 1)], call, chain) for keyword in call.keywords: if keyword.arg in _VALVE_KEYWORDS: - self._report(keyword.value, _VALVE_KEYWORDS[keyword.arg], call, chain, in_main) + self._report(keyword.value, _VALVE_KEYWORDS[keyword.arg], call, chain) - def _report(self, expr: ast.AST, role: str, call: ast.Call, chain: List[Dict[str, Binding]], - in_main: bool) -> None: - got = self._classify(expr, chain, in_main, set()) + def _report(self, expr: ast.AST, role: str, call: ast.Call, chain: List[Dict[str, Binding]]) -> None: + got = self._classify(expr, chain, set()) if got is None and role == "Valve predicate" and isinstance(expr, ast.Name) \ and self._resolve(expr.id, chain)[0] == "module_def": got = FUNCTION_PREDICATE, "`%s` is a function, not a predicate object" % expr.id @@ -328,3 +246,36 @@ def _report(self, expr: ast.AST, role: str, call: ast.Call, chain: List[Dict[str self._seen.add(key) self._findings.append(make_finding("python", self._path, expr.lineno, expr.col_offset + 1, rule, role, what, call.lineno)) + + @staticmethod + def _is_plain_pipeline_call(node: ast.AST) -> bool: + if not isinstance(node, ast.Call) or node.args or node.keywords: + return False + func = node.func + return (isinstance(func, ast.Name) and func.id == "Pipeline") or ( + isinstance(func, ast.Attribute) and func.attr == "Pipeline") + + @staticmethod + def _own_nodes(body: List[ast.stmt]): + """Every node in ``body``, not descending into nested functions, classes or lambdas.""" + stack = list(reversed(body)) + while stack: + node = stack.pop() + yield node + for child in reversed(list(ast.iter_child_nodes(node))): + if isinstance(child, _FUNCS + (ast.ClassDef, ast.Lambda)): + yield child # the definition itself binds a name; its body is its own scope + continue + stack.append(child) + + @staticmethod + def _target_names(target: ast.AST): + if isinstance(target, ast.Name): + yield target.id + elif isinstance(target, (ast.Tuple, ast.List)): + for element in target.elts: + for name in PythonStepVisitor._target_names(element): + yield name + elif isinstance(target, ast.Starred): + for name in PythonStepVisitor._target_names(target.value): + yield name diff --git a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/rules.py b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/rules.py index 4b3c0d9..84abb75 100644 --- a/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/rules.py +++ b/packages/codeupipe-lint/codeupipe_lint/anonymous_steps/rules.py @@ -1,7 +1,7 @@ """Rule ids, and the messages that name the fix (packages/codeupipe-lint/README.md). -The wording mirrors the runtime's AnonymousStep refusals -(packages/codeupipe/codeupipe/core/step_identity.py). +The runtime reports a step by the name of its class +(packages/codeupipe/codeupipe/core/step_label.py). """ from typing import Optional @@ -9,13 +9,12 @@ from .finding import Finding __all__ = ["RULES", "LAMBDA", "LOCAL_FUNCTION", "LOCAL_TYPE", "UNNAMED_PIPELINE", - "ENTRY_SCRIPT", "FUNCTION_PREDICATE", "PARSE_ERROR", "make_finding"] + "FUNCTION_PREDICATE", "PARSE_ERROR", "make_finding"] LAMBDA = "CUP-A001" LOCAL_FUNCTION = "CUP-A002" LOCAL_TYPE = "CUP-A003" UNNAMED_PIPELINE = "CUP-A005" -ENTRY_SCRIPT = "CUP-A006" FUNCTION_PREDICATE = "CUP-A008" PARSE_ERROR = "CUP-A000" @@ -25,11 +24,10 @@ LOCAL_FUNCTION: "local-function-step", LOCAL_TYPE: "local-type-step", UNNAMED_PIPELINE: "unnamed-pipeline-step", - ENTRY_SCRIPT: "entry-script-step", FUNCTION_PREDICATE: "function-predicate", } -_TAIL = " so it has a stable identity and can be tested on its own." +_TAIL = " so it has a name in State and can be tested on its own." # Steps and Valve predicates are named types (predicates are objects with # test(payload), conformance/API.md, "Valve"). @@ -42,21 +40,21 @@ } +# The rules that have a fix of their own. {pipeline} is the example of a named pipeline in the language. +_FIX_OF_RULE = { + UNNAMED_PIPELINE: ("A nested pipeline must be a named pipeline type ({pipeline}), named like a filter. " + "A plain Pipeline can only run at the top level."), + FUNCTION_PREDICATE: ("Valve predicates are objects (conformance/API.md, \"Valve\"): wrap the condition in a " + "named class with test(payload), configuration on the instance, and pass an instance."), + PARSE_ERROR: "Fix the syntax error so the file can be checked.", +} + + def _fix(lang: str, rule: str) -> str: - if rule == UNNAMED_PIPELINE: - return ("A nested pipeline must be a named pipeline type (%s), identified like a filter. " - "A plain Pipeline can only run at the top level." % _PIPELINE_EXAMPLE[lang]) - if rule == ENTRY_SCRIPT: - return ("Its identity would be __main__.X, which changes with how the program is started, " - "and tests cannot import it. Move it into a module.") - if rule == FUNCTION_PREDICATE: - return ("Valve predicates are objects (conformance/API.md, \"Valve\"): wrap the condition in a " - "named class with test(payload), configuration on the instance, and pass an instance.") - if rule == PARSE_ERROR: - return "Fix the syntax error so the file can be checked." - return _FIX[lang] + return _FIX_OF_RULE.get(rule, _FIX[lang]).replace("{pipeline}", _PIPELINE_EXAMPLE[lang]) +# cup: allow-plain it makes one Finding from its parts, for each analysis Filter def make_finding(lang: str, path: str, line: int, col: int, rule: str, role: str, what: str, call_line: Optional[int] = None) -> Finding: """A Finding whose message reads `` . ``; ``what`` starts with its verb diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/__main__.py b/packages/codeupipe-lint/codeupipe_lint/invariants/__main__.py index fba6d0a..61db748 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/__main__.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/__main__.py @@ -1,8 +1,8 @@ -"""``python -m codeupipe_lint.invariants [--exclude GLOB] [--config FILE] [--rules] [paths...]`` +"""``python -m codeupipe_lint.invariants [--exclude GLOB] [--strict] [--config FILE] [--rules] [paths...]`` With no paths, they come from ``cup.config.json`` (packages/codeupipe/codeupipe/config.py). -Exit code: 0 clean, 1 invariant violations found, 2 a path could not be read +Exit code: 0 clean, 1 invariant violations found (with ``--strict``, warnings too), 2 a path could not be read or the config cannot be used. """ @@ -16,6 +16,7 @@ from ..rule_ids import unknown_rules +# cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result def main(argv=None) -> int: parser = argparse.ArgumentParser(prog="python -m codeupipe_lint.invariants", description="Enforce CUP's structural invariants (packages/codeupipe-lint/README.md).") @@ -24,6 +25,7 @@ def main(argv=None) -> int: help="skip files whose path matches this fnmatch glob (repeatable)") parser.add_argument("--disable", action="append", metavar="RULE", help="switch this rule off (repeatable), e.g. CUP-I005") + parser.add_argument("--strict", action="store_true", help="a warning fails the run, as a finding does") parser.add_argument("--config", metavar="FILE", help="read paths and excludes from FILE, not %s" % DEFAULT_NAME) parser.add_argument("--rules", action="store_true", help="list the rule ids and exit") args = parser.parse_args(argv) @@ -49,7 +51,7 @@ def main(argv=None) -> int: print("error: no such rule: %s (the rules are listed by --rules of each linter)" % ", ".join(unknown), file=sys.stderr) return 2 - out = lint_paths(paths, exclude, disable) + out = lint_paths(paths, exclude, disable, strict=args.strict) print(out.get("report")) if out.get("error_report"): print(out.get("error_report"), file=sys.stderr) diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_plain_logic_filter.py b/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_plain_logic_filter.py new file mode 100644 index 0000000..726e9e4 --- /dev/null +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_plain_logic_filter.py @@ -0,0 +1,69 @@ +"""AnalyzePlainLogicFilter: CUP-I008, a warning: logic that is not in a primitive. + +A project that uses codeupipe writes its logic as Filters, Predicates, Taps, +Hooks and Pipelines. Logic in a plain function or a plain class works as +well, so nothing else shows the mistake. This rule shows it. + +Logic is a decision or a loop: ``if``, ``a if c else b``, ``match``, +``for``, ``while``, ``try``. Flagged, once each: a function at the top of a +file, a class at the top of a file that is not a primitive, and the code at +the top of a file itself. Not flagged: a primitive (its logic is where it +must be), a function or a class with no logic, a test file, and the guards +``if __name__ == "__main__":`` and ``if TYPE_CHECKING:``. Not seen: a +comprehension, and ``and`` / ``or``. +""" + +import ast + +from codeupipe import Payload + +from .primitive_kind import is_test_file, primitive_kind +from .rules import LOGIC_OUTSIDE_PRIMITIVE, make_finding + +_WHAT = "has logic and is not a Filter, a Predicate, a Tap, a Hook or a Pipeline" +# ast.Match is new in Python 3.10 and ast.TryStar in 3.11. +_LOGIC = tuple(getattr(ast, name) for name in ("If", "IfExp", "For", "AsyncFor", "While", "Try", "TryStar", "Match") + if hasattr(ast, name)) +_GUARDS = ("__name__", "TYPE_CHECKING") + + +class AnalyzePlainLogicFilter: + """Filter (sync). + + Input keys: ``trees`` ({path: ast.Module}), ``findings`` (list). + Output keys: ``findings`` with CUP-I008 warnings appended. + """ + + def call(self, payload: Payload) -> Payload: + findings = list(payload.get("findings", [])) + for path, tree in payload.get("trees", {}).items(): + if is_test_file(path): + continue + module_code = None + for node in tree.body: + what = None + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + what = "Function `%s`" % node.name if self.has_logic(node.body) else None + elif isinstance(node, ast.ClassDef): + if primitive_kind(node) is None and self.has_logic(node.body): + what = "Class `%s`" % node.name + elif module_code is None and self.has_logic(self.without_guard(node)): + module_code = node + if what is not None: + findings.append(make_finding(path, node.lineno, node.col_offset + 1, LOGIC_OUTSIDE_PRIMITIVE, + "%s %s" % (what, _WHAT))) + if module_code is not None: + findings.append(make_finding(path, module_code.lineno, module_code.col_offset + 1, + LOGIC_OUTSIDE_PRIMITIVE, "The code at the top of this file %s" % _WHAT)) + return payload.insert("findings", findings) + + def has_logic(self, nodes) -> bool: + return any(isinstance(inner, _LOGIC) for node in nodes for inner in ast.walk(node)) + + def without_guard(self, node): + """The statements to look in: for a guard of the language, those in it, and not the ``if`` itself.""" + if isinstance(node, ast.If) and not node.orelse and any( + isinstance(name, ast.Name) and name.id in _GUARDS or isinstance(name, ast.Attribute) and name.attr in _GUARDS + for name in ast.walk(node.test)): + return node.body + return [node] diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_stale_markers_filter.py b/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_stale_markers_filter.py new file mode 100644 index 0000000..7c5e5a1 --- /dev/null +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/analyze_stale_markers_filter.py @@ -0,0 +1,57 @@ +"""AnalyzeStaleMarkersFilter: CUP-I009, a warning: a comment that allows nothing. + +A ``# cup: allow-...`` comment that allowed no finding stays when the code +that it was for is gone or has moved, or when its kind has a spelling error. +It then says that the code has an exception that it does not have. + +Only a comment that starts with the marker counts, so text that names a +marker in a string or in the middle of a sentence is not reported. A +``cup: allow-anonymous`` comment is for the other linter +(``codeupipe_lint.anonymous_steps``) and is not looked at here. +""" + +import io +import re +import tokenize + +from codeupipe import Payload + +from .rules import MARKER_ALLOWS_NOTHING, make_finding + +_MARKER = re.compile(r"#+\s*cup:\s*(allow-[\w-]*)") +_OTHER_LINTER = ("allow-anonymous",) + + +class AnalyzeStaleMarkersFilter: + """Filter (sync). + + Input keys: ``trees`` ({path: ast.Module}), ``findings``, ``used_markers`` + (from ApplySuppressionsFilter). + Output keys: ``findings`` with CUP-I009 warnings appended. + """ + + def call(self, payload: Payload) -> Payload: + used = {tuple(place) for place in payload.get("used_markers", [])} + findings = list(payload.get("findings", [])) + for path in payload.get("trees", {}): + for line, col, kind in self.markers_of(path): + if (path, line) in used or kind in _OTHER_LINTER: + continue + findings.append(make_finding(path, line, col, MARKER_ALLOWS_NOTHING, + "The comment `cup: %s` allows nothing" % kind)) + return payload.insert("findings", findings) + + def markers_of(self, path): + """``(line, col, kind)`` for each comment of the file that starts with a marker.""" + try: + with open(path, encoding="utf-8", errors="replace") as handle: + text = handle.read() + tokens = list(tokenize.generate_tokens(io.StringIO(text).readline)) + except (OSError, tokenize.TokenError, SyntaxError): + return [] + markers = [] + for token in tokens: + match = _MARKER.match(token.string) if token.type == tokenize.COMMENT else None + if match: + markers.append((token.start[0], token.start[1] + 1, match.group(1))) + return markers diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/lint_pipeline.py b/packages/codeupipe-lint/codeupipe_lint/invariants/lint_pipeline.py index d625bb0..ee3e1e1 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/lint_pipeline.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/lint_pipeline.py @@ -7,12 +7,15 @@ 4. AnalyzeStateReadsFilter CUP-I003 5. AnalyzeOutsideWritesFilter CUP-I004 6. AnalyzePrimitivesPerFileFilter CUP-I005 - 7. AnalyzePrimitiveNamesFilter CUP-I006, CUP-I007 - 8. DropDisabledFilter rules switched off for the project + 7. AnalyzePrimitiveNamesFilter CUP-I006, CUP-I007 + 8. AnalyzePlainLogicFilter CUP-I008, a warning 9. ApplySuppressionsFilter ``cup: allow-mutation``, ``cup: allow-state-read``, - ``cup: allow-outside-write``, ``cup: allow-shared-file``, - ``cup: allow-name`` - 10. FormatReportFilter report lines + exit code + ``cup: allow-outside-write``, ``cup: allow-shared-file``, + ``cup: allow-name``, ``cup: allow-plain`` + 10. AnalyzeStaleMarkersFilter CUP-I009, a warning + 11. DropDisabledFilter rules switched off for the project. Last, so that a + comment for a rule that is off is not stale + 12. FormatReportFilter report lines + exit code """ import re @@ -27,11 +30,13 @@ from .analyze_outside_writes_filter import AnalyzeOutsideWritesFilter from .analyze_primitive_names_filter import AnalyzePrimitiveNamesFilter from .analyze_payload_mutation_filter import AnalyzePayloadMutationFilter +from .analyze_plain_logic_filter import AnalyzePlainLogicFilter +from .analyze_stale_markers_filter import AnalyzeStaleMarkersFilter from .analyze_primitives_per_file_filter import AnalyzePrimitivesPerFileFilter from .analyze_state_reads_filter import AnalyzeStateReadsFilter from .parse_python_filter import ParsePythonFilter -MARKER = re.compile(r"cup:\s*allow-(mutation|state-read|outside-write|shared-file|name)\b") +MARKER = re.compile(r"cup:\s*allow-(mutation|state-read|outside-write|shared-file|name|plain)\b") def build_lint_pipeline() -> Pipeline: @@ -43,19 +48,18 @@ def build_lint_pipeline() -> Pipeline: pipeline.add_filter(AnalyzeOutsideWritesFilter()) pipeline.add_filter(AnalyzePrimitivesPerFileFilter()) pipeline.add_filter(AnalyzePrimitiveNamesFilter()) - pipeline.add_filter(DropDisabledFilter()) + pipeline.add_filter(AnalyzePlainLogicFilter()) pipeline.add_filter(ApplySuppressionsFilter(MARKER)) + pipeline.add_filter(AnalyzeStaleMarkersFilter()) + pipeline.add_filter(DropDisabledFilter()) pipeline.add_filter(FormatReportFilter("invariant violation", "invariant violations")) return pipeline def lint_paths(paths: Iterable[str], exclude: Optional[Iterable[str]] = None, - disable: Optional[Iterable[str]] = None) -> Payload: + disable: Optional[Iterable[str]] = None, strict: bool = False) -> Payload: """Run the linter over ``paths``; the result Payload holds ``findings``, ``suppressed``, ``errors``, ``report``, ``error_report`` and ``exit_code``.""" - data = {"paths": list(paths), "findings": [], "errors": []} - if exclude: - data["exclude"] = list(exclude) - if disable: - data["disable"] = list(disable) + data = {"paths": list(paths), "findings": [], "errors": [], + "exclude": list(exclude or ()), "disable": list(disable or ()), "strict": bool(strict)} return build_lint_pipeline().run(Payload(data)) diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/outside_write_visitor.py b/packages/codeupipe-lint/codeupipe_lint/invariants/outside_write_visitor.py index 7ebafe3..97df595 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/outside_write_visitor.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/outside_write_visitor.py @@ -19,6 +19,7 @@ WRITES = {"insert", "remove", "merge", "with_trace"} +# cup: allow-plain used by the analysis Filters and the visitor: it finds the payload parameter of a method def payload_parameter(method): """The parameter that holds the Payload: the one named ``payload``, else the last one.""" names = [a.arg for a in method.args.posonlyargs + method.args.args] @@ -29,6 +30,7 @@ def payload_parameter(method): return "payload" if "payload" in names else names[-1] +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code class OutsideWriteVisitor(ast.NodeVisitor): def __init__(self, path, method): self.path = path diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/payload_mutation_visitor.py b/packages/codeupipe-lint/codeupipe_lint/invariants/payload_mutation_visitor.py index 47649e7..ec07af5 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/payload_mutation_visitor.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/payload_mutation_visitor.py @@ -27,13 +27,7 @@ "symmetric_difference_update", "appendleft", "extendleft", "popleft"} -def _base(node): - """The name or call an attribute/subscript chain hangs off.""" - while isinstance(node, (ast.Subscript, ast.Attribute)): - node = node.value - return node - - +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code class PayloadMutationVisitor(ast.NodeVisitor): def __init__(self, path, method): self.path = path @@ -90,7 +84,7 @@ def _check_write_target(self, target, statement): if self._is_internals(chain): self._report(statement, PAYLOAD_INTERNALS, "Writes a Payload's internals (`_data`)") return - base = _base(target) + base = PayloadMutationVisitor._base(target) if isinstance(target, ast.Subscript) and (self._is_get(base) or (isinstance(base, ast.Name) and base.id in self.shared)): self._report(statement, SHARED_VALUE_MUTATION, self._describe(base)) @@ -146,7 +140,7 @@ def visit_Call(self, node): if self._is_internals(receiver): self._report(node, PAYLOAD_INTERNALS, "Writes a Payload's internals (`_data`)") else: - base = _base(receiver) + base = PayloadMutationVisitor._base(receiver) if self._is_get(base) or (isinstance(base, ast.Name) and base.id in self.shared): self._report(node, SHARED_VALUE_MUTATION, self._describe(base)) self.generic_visit(node) @@ -155,3 +149,10 @@ def _skip(self, node): """A nested function or class is its own scope; step methods are found from the module.""" visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = visit_Lambda = _skip + + @staticmethod + def _base(node): + """The name or call an attribute/subscript chain hangs off.""" + while isinstance(node, (ast.Subscript, ast.Attribute)): + node = node.value + return node diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/primitive_kind.py b/packages/codeupipe-lint/codeupipe_lint/invariants/primitive_kind.py index 602c601..8595861 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/primitive_kind.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/primitive_kind.py @@ -23,6 +23,7 @@ _HOOK_BASES = {"Hook", "AsyncHook"} +# cup: allow-plain shared by the analysis Filters: it says which primitive a class is, from its shape def _base_names(node): return {base.id if isinstance(base, ast.Name) else getattr(base, "attr", "") for base in node.bases} @@ -33,6 +34,7 @@ def _takes_only_the_payload(method) -> bool: and not method.args.vararg and not method.args.kwonlyargs and not method.args.kwarg) +# cup: allow-plain shared by the analysis Filters: it says which primitive a class is, from its shape def primitive_kind(node): """"Pipeline", "Hook", "Filter", "Tap" or "predicate"; None for any other class.""" methods = {member.name: member for member in node.body diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/rules.py b/packages/codeupipe-lint/codeupipe_lint/invariants/rules.py index 07ce597..2adbfe7 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/rules.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/rules.py @@ -4,7 +4,7 @@ __all__ = ["RULES", "PARSE_ERROR", "PAYLOAD_INTERNALS", "SHARED_VALUE_MUTATION", "STATE_READ", "WRITE_OUTSIDE_FILTER", "ONE_PRIMITIVE_PER_FILE", "PRIMITIVE_FILE_NAME", - "PRIMITIVE_ROLE_SUFFIX", "make_finding"] + "PRIMITIVE_ROLE_SUFFIX", "LOGIC_OUTSIDE_PRIMITIVE", "MARKER_ALLOWS_NOTHING", "WARNINGS", "make_finding"] PARSE_ERROR = "CUP-I000" PAYLOAD_INTERNALS = "CUP-I001" @@ -14,6 +14,12 @@ ONE_PRIMITIVE_PER_FILE = "CUP-I005" PRIMITIVE_FILE_NAME = "CUP-I006" PRIMITIVE_ROLE_SUFFIX = "CUP-I007" +LOGIC_OUTSIDE_PRIMITIVE = "CUP-I008" +MARKER_ALLOWS_NOTHING = "CUP-I009" + +# The rules whose findings are warnings. +WARNINGS = frozenset([LOGIC_OUTSIDE_PRIMITIVE, MARKER_ALLOWS_NOTHING]) +_SEVERITY = {rule: "warning" for rule in WARNINGS} RULES = { PARSE_ERROR: "parse-error", @@ -24,6 +30,8 @@ ONE_PRIMITIVE_PER_FILE: "one-primitive-per-file", PRIMITIVE_FILE_NAME: "primitive-file-name", PRIMITIVE_ROLE_SUFFIX: "primitive-role-suffix", + LOGIC_OUTSIDE_PRIMITIVE: "logic-outside-primitive", + MARKER_ALLOWS_NOTHING: "marker-allows-nothing", } _FIX = { @@ -43,9 +51,14 @@ "the other."), PRIMITIVE_ROLE_SUFFIX: ("A primitive's class name ends with its role (Filter, Tap, Hook, Predicate, " "Pipeline), so the name tells what the class is."), + LOGIC_OUTSIDE_PRIMITIVE: ("Write it as a primitive, so that the pipeline has the logic and it can be tested " + "as a step. For logic that must be plain code, say why: # cup: allow-plain "), + MARKER_ALLOWS_NOTHING: ("Remove it, or put it on the line of the finding that it is for. A comment that " + "allows nothing says that the code has an exception that it does not have."), } def make_finding(path: str, line: int, col: int, rule: str, what: str) -> Finding: """A Finding whose message reads ``. ``.""" - return Finding(path, line, col, rule, RULES[rule], "%s. %s" % (what, _FIX[rule]), "python") + return Finding(path, line, col, rule, RULES[rule], "%s. %s" % (what, _FIX[rule]), "python", + severity=_SEVERITY.get(rule)) diff --git a/packages/codeupipe-lint/codeupipe_lint/invariants/state_read_visitor.py b/packages/codeupipe-lint/codeupipe_lint/invariants/state_read_visitor.py index f24c811..e4fd210 100644 --- a/packages/codeupipe-lint/codeupipe_lint/invariants/state_read_visitor.py +++ b/packages/codeupipe-lint/codeupipe_lint/invariants/state_read_visitor.py @@ -24,23 +24,11 @@ "record_timing", "record_error"} -def _is_state_type(node) -> bool: - if isinstance(node, ast.Name): - return node.id == "State" - if isinstance(node, ast.Attribute): - return node.attr == "State" - return isinstance(node, ast.Constant) and node.value == "State" - - def _is_self(node) -> bool: return isinstance(node, ast.Name) and node.id == "self" -def _names_a_pipeline(node) -> bool: - name = node.id if isinstance(node, ast.Name) else node.attr if isinstance(node, ast.Attribute) else "" - return "pipeline" in name.lower() - - +# cup: allow-plain used by the analysis Filter and the visitor: it finds the names that hold a State def state_handles(class_node) -> set: """Attributes that ``__init__`` sets to a State.""" handles = set() @@ -48,19 +36,20 @@ def state_handles(class_node) -> set: if not (isinstance(member, ast.FunctionDef) and member.name == "__init__"): continue state_parameters = {a.arg for a in member.args.args + member.args.kwonlyargs - if a.annotation is not None and _is_state_type(a.annotation)} + if a.annotation is not None and StateReadVisitor._is_state_type(a.annotation)} for node in ast.walk(member): if not isinstance(node, ast.Assign): continue value = node.value is_state = ((isinstance(value, ast.Name) and value.id in state_parameters) or (isinstance(value, ast.Attribute) and value.attr == "state") - or (isinstance(value, ast.Call) and _is_state_type(value.func))) + or (isinstance(value, ast.Call) and StateReadVisitor._is_state_type(value.func))) if is_state: handles.update(t.attr for t in node.targets if isinstance(t, ast.Attribute) and _is_self(t.value)) return handles +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code class StateReadVisitor(ast.NodeVisitor): def __init__(self, path, method, handles): self.path = path @@ -75,7 +64,7 @@ def _reaches_state(self, node) -> bool: if _is_self(node.value) and node.attr in self.handles: return True if node.attr == "state" and not _is_self(node.value): - if outer in STATE_MEMBERS or _names_a_pipeline(node.value): + if outer in STATE_MEMBERS or StateReadVisitor._names_a_pipeline(node.value): return True outer = node.attr node = node.value @@ -92,3 +81,16 @@ def _skip(self, node): """A nested function or class is its own scope.""" visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = visit_Lambda = _skip + + @staticmethod + def _is_state_type(node) -> bool: + if isinstance(node, ast.Name): + return node.id == "State" + if isinstance(node, ast.Attribute): + return node.attr == "State" + return isinstance(node, ast.Constant) and node.value == "State" + + @staticmethod + def _names_a_pipeline(node) -> bool: + name = node.id if isinstance(node, ast.Name) else node.attr if isinstance(node, ast.Attribute) else "" + return "pipeline" in name.lower() diff --git a/packages/codeupipe-lint/pyproject.toml b/packages/codeupipe-lint/pyproject.toml index dd4d45c..658572e 100644 --- a/packages/codeupipe-lint/pyproject.toml +++ b/packages/codeupipe-lint/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "codeupipe-lint" -version = "1.0.0" +version = "2.0.0b1" description = "Lint rules for codeupipe pipelines." readme = "README.md" requires-python = ">=3.9" @@ -13,6 +13,7 @@ license-files = ["LICENSE", "NOTICE"] authors = [{name = "Joshua Wink"}] keywords = ["codeupipe", "lint", "pipeline"] classifiers = [ + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", @@ -25,7 +26,7 @@ classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Quality Assurance", ] -dependencies = ["codeupipe==1.0.0"] +dependencies = ["codeupipe==2.0.0b1"] [project.urls] Homepage = "https://github.com/orchestrate-solutions/codeupipe" diff --git a/packages/codeupipe-rs/Cargo.toml b/packages/codeupipe-rs/Cargo.toml new file mode 100644 index 0000000..cdf432a --- /dev/null +++ b/packages/codeupipe-rs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "codeupipe" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +description = "A Rust library for writing a program as a pipeline of small steps." +license = "Apache-2.0" +repository = "https://github.com/orchestrate-solutions/codeupipe" +readme = "README.md" +keywords = ["pipeline", "filter", "payload", "composition"] +# Not on crates.io yet. A version below 1.0 is not stable. +publish = false + +[[example]] +name = "pricing" +path = "examples/pricing/main.rs" + +[[example]] +name = "typed_pricing" +path = "examples/typed_pricing/main.rs" diff --git a/packages/codeupipe-rs/LICENSE b/packages/codeupipe-rs/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-rs/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-rs/NOTICE b/packages/codeupipe-rs/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-rs/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-rs/README.md b/packages/codeupipe-rs/README.md new file mode 100644 index 0000000..3e074ed --- /dev/null +++ b/packages/codeupipe-rs/README.md @@ -0,0 +1,224 @@ +# codeupipe for Rust + +codeupipe is a library for writing a program as a pipeline of small steps. +This is the Rust runtime. It has no dependencies. + +It implements the same contract as the Python and TypeScript packages: +[conformance/API.md](../../conformance/API.md), and passes all its +conformance cases (`python conformance/run.py rust:run`). The linter for +Rust is [packages/codeupipe-lint-rs/](../codeupipe-lint-rs/); a compiler is +not planned yet ([docs/rust-plan.md](../../docs/rust-plan.md)). + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. + +The crate is not on crates.io yet. Use it from this repository: + +```toml +[dependencies] +codeupipe = { git = "https://github.com/orchestrate-solutions/codeupipe", branch = "main" } +``` + +## The eight primitives + +| Primitive | What it does | Who writes it | +|---|---|---| +| **Payload** | Carries the data | The library | +| **Filter** | Changes the data | You | +| **Predicate** | Asks a yes-or-no question about the data | You | +| **Valve** | Joins a Predicate to a Filter: the Filter runs when the answer is yes | The library. You add one with `add_valve(predicate, filter)` | +| **Tap** | Reads the data as it goes by | You | +| **Hook** | Is called before and after each step | You | +| **State** | Records which steps ran | The library | +| **Pipeline** | Runs the steps in order | You | + +## Example + +[examples/pricing/](examples/pricing/) has one type in each file. The +pipeline file adds them. + +```rust +// discount_filter.rs +use codeupipe::{Filter, Payload, StepError, Value}; + +pub struct DiscountFilter { + rate: f64, +} + +impl Filter for DiscountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) * (1.0 - self.rate); + payload.insert("price", (price * 100.0).round() / 100.0); + Ok(()) + } +} +``` + +```rust +// at_least_predicate.rs +use codeupipe::{Payload, Predicate, Value}; + +pub struct AtLeastPredicate { + key: String, + minimum: f64, +} + +impl Predicate for AtLeastPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get(&self.key).and_then(Value::as_f64).unwrap_or(0.0) >= self.minimum + } +} +``` + +```rust +// pricing_pipeline.rs +use codeupipe::{NamedPipeline, Pipeline}; + +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} +``` + +```rust +use codeupipe::{Map, Payload, Pipeline}; + +let pipeline = Pipeline::named(PricingPipeline); +let result = pipeline.run(&Payload::from_map(Map::from([("price", 100.0), ("qty", 12.0)])))?; +result.to_dict(); +// {"price": 90, "qty": 12, "bulk": true} +``` + +```bash +cargo run --example pricing +``` + +## A payload of your own + +`Payload` is the default, and it stays the default: a map of JSON-shaped +values that any developer can read, in any runtime of codeupipe. It shows +that anything can go there. + +You can opt in to a type of your own, such as the struct that you would +write anyway. Implement `PipelineData` for it (one line), and the steps of +that pipeline take the struct: + +```rust +#[derive(Clone)] +pub struct Order { + pub price: f64, + pub qty: u32, + pub bulk: bool, +} + +impl PipelineData for Order {} + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + pipeline.add_valve(AtLeastPredicate::new(10), MarkBulkFilter); + pipeline.add_tap(AuditTap); + } +} +``` + +`Filter`, `Predicate`, `Tap`, `Hook`, `NamedPipeline` and `Pipeline` each take +the type as a parameter, which is `Payload` when you leave it out. The rules +stay the same: `run` works on a clone of the input, only a Filter writes, a +Tap failure is recorded and the run goes on, and the labels are the same. A +step for another type is a compile error. Lineage is kept only if the type +implements `PipelineData::stamp`. + +The whole example is in [examples/typed_pricing/](examples/typed_pricing/): + +```bash +cargo run --example typed_pricing +``` + +A typed pipeline has no map to look keys up in, so it is faster: on the +pricing pipeline 0.024 µs a run against 0.18 µs with a Payload, and for 50 +writes 0.46 µs against 4.3 µs (Apple Silicon, release build). + +## Names + +The names are the names of the contract. + +| Contract | Rust | +|---|---| +| `add_filter`, `add_valve`, `add_tap`, `add_pipeline`, `use_hook` | the same | +| `run(p)`, `run(p, state)`, `state`, `labels` | `run(&p)`, `run_with_state(&p, &mut state)`, `state()`, `labels()` | +| `observe(timing, lineage)` | `observe(timing, lineage)`, two `bool`s | +| `get`, `has`, `insert`, `remove`, `merge`, `copy`, `to_dict`, `with_trace` | the same | +| `trace_id`, `lineage` | `trace_id()`, `lineage()` | +| Hook: `before`, `after`, `on_error` | the same | +| A named pipeline: `class PricingPipeline(Pipeline)` | `impl NamedPipeline for PricingPipeline`, then `Pipeline::named(PricingPipeline)` | + +`Filter`, `Predicate`, `Tap` and `Hook` are traits. The methods of `Hook` do +nothing unless you implement them. + +## Differences from Python and TypeScript + +- **A Filter gets `&mut Payload`.** It writes in place, as the Payload is + changed in place in TypeScript, and returns `Ok(())`. A Tap, a Predicate and + a Hook get `&Payload`, so **only a Filter can write, and the compiler checks + it**. A Hook's `on_error` sees the Payload as the failing step left it. +- **What the contract refuses when a step is added is a compile error.** A + closure is not a Filter, and `add_valve(filter, predicate)` does not + type-check. The doc tests in `src/lib.rs` check this. +- **`get` gives `Option<&Value>`.** Absent is `None`. A default is + `.unwrap_or(...)`. `Value` is JSON-shaped, with one number type: `1` and + `1.0` are the same value. +- **`run` is sync, and there is no `run_async` yet.** It takes `&self`, so a + step can hold the pipeline that it is in (an `Rc`) and run it again; each + run writes into a State of its own. Rust has no optional argument, so the + run with the caller's State is `run_with_state(&input, &mut state)`. +- **`state()` gives an `Rc`**: the State of the run that started last, + as a snapshot that a later run does not change. Keep it in a variable to + borrow from it (`let state = pipeline.state();`). Its errors are + `Arc` (`SharedError`), so that a State can be cloned. +- **A failure is an `Err`.** A panic is not a failure: it is not caught, and + a panicking Tap stops the run. +- **`copy()` is deep.** A value shared by two Payloads could not be changed in + place anyway, so the behaviour is the same as the shallow copy of the + contract. +- **The label** is the type name without its path or generic arguments. It + comes from `std::any::type_name`, whose exact text Rust does not promise to + keep the same; the tests check it. +- **A payload of your own** is a type parameter of each primitive (see + above), as in TypeScript. Python takes any object with `copy()`. +- **Timing in WebAssembly.** `std::time::Instant` panics on + `wasm32-unknown-unknown`. Timing is off by default, so a pipeline runs + there as long as `observe(true, ...)` is not called. + +## Development + +The Rust packages are one Cargo workspace. At the root of the repository: + +```bash +cargo test # the tests, and the doc tests +cargo clippy -- -D warnings +cargo fmt --check +cargo run --example pricing +cargo run --example typed_pricing +``` + +One primitive is in each file, and the file name is the type name in +snake_case. + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs b/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs new file mode 100644 index 0000000..4592f4c --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/at_least_predicate.rs @@ -0,0 +1,27 @@ +use codeupipe::{Payload, Predicate, Value}; + +/// A Predicate: asks a yes-or-no question and changes nothing. Its settings +/// are on the instance. +pub struct AtLeastPredicate { + key: String, + minimum: f64, +} + +impl AtLeastPredicate { + pub fn new(key: &str, minimum: f64) -> Self { + AtLeastPredicate { + key: key.to_string(), + minimum, + } + } +} + +impl Predicate for AtLeastPredicate { + fn test(&self, payload: &Payload) -> bool { + payload + .get(&self.key) + .and_then(Value::as_f64) + .unwrap_or(0.0) + >= self.minimum + } +} diff --git a/packages/codeupipe-rs/examples/pricing/audit_tap.rs b/packages/codeupipe-rs/examples/pricing/audit_tap.rs new file mode 100644 index 0000000..28d3079 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/audit_tap.rs @@ -0,0 +1,12 @@ +use codeupipe::{Payload, StepError, Tap}; + +/// A Tap: reads the Payload and returns nothing. If it fails, State records +/// the error and the run goes on. +pub struct AuditTap; + +impl Tap for AuditTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + println!("audit: {:?}", payload.to_dict()); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/discount_filter.rs b/packages/codeupipe-rs/examples/pricing/discount_filter.rs new file mode 100644 index 0000000..f1145cd --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/discount_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// A Filter. Its setting is on the instance. +pub struct DiscountFilter { + rate: f64, +} + +impl DiscountFilter { + pub fn new(rate: f64) -> Self { + DiscountFilter { rate } + } +} + +impl Filter for DiscountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) * (1.0 - self.rate); + payload.insert("price", (price * 100.0).round() / 100.0); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/main.rs b/packages/codeupipe-rs/examples/pricing/main.rs new file mode 100644 index 0000000..527c99c --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/main.rs @@ -0,0 +1,35 @@ +//! cargo run --example pricing: run the pipeline with a Hook and show its State. +mod at_least_predicate; +mod audit_tap; +mod discount_filter; +mod mark_bulk_filter; +mod pricing_pipeline; +mod print_steps_hook; + +use codeupipe::{Map, Payload, Pipeline, StepError}; + +use pricing_pipeline::PricingPipeline; +use print_steps_hook::PrintStepsHook; + +fn main() -> Result<(), StepError> { + let mut pipeline = Pipeline::named(PricingPipeline); + pipeline.use_hook(PrintStepsHook); + + println!("steps:"); + let result = pipeline.run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 12.0), + ])))?; + + let state = pipeline.state(); + let errors: Vec<(&str, String)> = state + .errors() + .iter() + .map(|(label, error)| (label.as_str(), error.to_string())) + .collect(); + println!("result: {:?}", result.to_dict()); + println!("executed: {:?}", state.executed()); + println!("skipped: {:?}", state.skipped()); + println!("errors: {errors:?}"); + Ok(()) +} diff --git a/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs b/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs new file mode 100644 index 0000000..3cba368 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/mark_bulk_filter.rs @@ -0,0 +1,11 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// A Filter. +pub struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("bulk", true); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs b/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs new file mode 100644 index 0000000..901db31 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/pricing_pipeline.rs @@ -0,0 +1,19 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::at_least_predicate::AtLeastPredicate; +use crate::audit_tap::AuditTap; +use crate::discount_filter::DiscountFilter; +use crate::mark_bulk_filter::MarkBulkFilter; + +/// A named pipeline: `Pipeline::named(PricingPipeline)`. +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} diff --git a/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs b/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs new file mode 100644 index 0000000..3132b99 --- /dev/null +++ b/packages/codeupipe-rs/examples/pricing/print_steps_hook.rs @@ -0,0 +1,13 @@ +use codeupipe::{Hook, Payload}; + +/// A Hook: sees the lifecycle. `step` is the label of the step, which is the +/// name of its type. It is `None` for the pipeline itself. +pub struct PrintStepsHook; + +impl Hook for PrintStepsHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + if let Some(step) = step { + println!(" {step}"); + } + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs b/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs new file mode 100644 index 0000000..d53350e --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/at_least_predicate.rs @@ -0,0 +1,21 @@ +use codeupipe::Predicate; + +use crate::order::Order; + +/// A Predicate: asks a yes-or-no question and changes nothing. Its setting +/// is on the instance. +pub struct AtLeastPredicate { + minimum: u32, +} + +impl AtLeastPredicate { + pub fn new(minimum: u32) -> Self { + AtLeastPredicate { minimum } + } +} + +impl Predicate for AtLeastPredicate { + fn test(&self, order: &Order) -> bool { + order.qty >= self.minimum + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs b/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs new file mode 100644 index 0000000..602885d --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/audit_tap.rs @@ -0,0 +1,14 @@ +use codeupipe::{StepError, Tap}; + +use crate::order::Order; + +/// A Tap: reads the Order and returns nothing. If it fails, State records +/// the error and the run goes on. +pub struct AuditTap; + +impl Tap for AuditTap { + fn observe(&self, order: &Order) -> Result<(), StepError> { + println!("audit: {order:?}"); + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs b/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs new file mode 100644 index 0000000..c6eafcd --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/discount_filter.rs @@ -0,0 +1,21 @@ +use codeupipe::{Filter, StepError}; + +use crate::order::Order; + +/// A Filter. Its setting is on the instance. +pub struct DiscountFilter { + rate: f64, +} + +impl DiscountFilter { + pub fn new(rate: f64) -> Self { + DiscountFilter { rate } + } +} + +impl Filter for DiscountFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.price = (order.price * (1.0 - self.rate) * 100.0).round() / 100.0; + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/main.rs b/packages/codeupipe-rs/examples/typed_pricing/main.rs new file mode 100644 index 0000000..1141f9d --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/main.rs @@ -0,0 +1,24 @@ +//! cargo run --example typed_pricing: the pricing example with a struct of +//! your own in place of the Payload. +mod at_least_predicate; +mod audit_tap; +mod discount_filter; +mod mark_bulk_filter; +mod order; +mod pricing_pipeline; + +use codeupipe::{Pipeline, StepError}; + +use order::Order; +use pricing_pipeline::PricingPipeline; + +fn main() -> Result<(), StepError> { + let pipeline = Pipeline::named(PricingPipeline); + let result = pipeline.run(&Order::new(100.0, 12))?; + + let state = pipeline.state(); + println!("result: {result:?}"); + println!("executed: {:?}", state.executed()); + println!("skipped: {:?}", state.skipped()); + Ok(()) +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs b/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs new file mode 100644 index 0000000..bdd8aa3 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/mark_bulk_filter.rs @@ -0,0 +1,13 @@ +use codeupipe::{Filter, StepError}; + +use crate::order::Order; + +/// A Filter. +pub struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} diff --git a/packages/codeupipe-rs/examples/typed_pricing/order.rs b/packages/codeupipe-rs/examples/typed_pricing/order.rs new file mode 100644 index 0000000..a24fe35 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/order.rs @@ -0,0 +1,23 @@ +use codeupipe::PipelineData; + +/// The data of this pipeline: a struct in place of the Payload. Each step +/// below takes an `Order`, and the compiler checks every field it uses. +#[derive(Clone, Debug, PartialEq)] +pub struct Order { + pub price: f64, + pub qty: u32, + pub bulk: bool, +} + +impl Order { + pub fn new(price: f64, qty: u32) -> Self { + Order { + price, + qty, + bulk: false, + } + } +} + +/// The one line that lets a Pipeline carry an Order. +impl PipelineData for Order {} diff --git a/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs b/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs new file mode 100644 index 0000000..95ab703 --- /dev/null +++ b/packages/codeupipe-rs/examples/typed_pricing/pricing_pipeline.rs @@ -0,0 +1,20 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::at_least_predicate::AtLeastPredicate; +use crate::audit_tap::AuditTap; +use crate::discount_filter::DiscountFilter; +use crate::mark_bulk_filter::MarkBulkFilter; +use crate::order::Order; + +/// The pricing pipeline of `examples/pricing`, carrying an Order. +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + // MarkBulkFilter runs only when the predicate is true. + pipeline.add_valve(AtLeastPredicate::new(10), MarkBulkFilter); + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.add_tap(AuditTap); + } +} diff --git a/packages/codeupipe-rs/src/filter.rs b/packages/codeupipe-rs/src/filter.rs new file mode 100644 index 0000000..384fc15 --- /dev/null +++ b/packages/codeupipe-rs/src/filter.rs @@ -0,0 +1,10 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Filter: changes the data. A failure is an `Err`, and it stops the run. +/// +/// The Filter gets the Payload itself and writes to it in place. It is the +/// only step that can write: the others get `&Payload`. +pub trait Filter

{ + fn call(&self, payload: &mut P) -> Result<(), StepError>; +} diff --git a/packages/codeupipe-rs/src/hook.rs b/packages/codeupipe-rs/src/hook.rs new file mode 100644 index 0000000..43776a1 --- /dev/null +++ b/packages/codeupipe-rs/src/hook.rs @@ -0,0 +1,18 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Hook: sees the lifecycle of a run. Implement the methods that you need; +/// the others do nothing. +/// +/// `step` is the label of the step, or `None` for the pipeline itself. A Hook +/// reads the Payload and does not write to it. +pub trait Hook

{ + /// Before a step, or before the pipeline starts (`step` is `None`). + fn before(&self, _step: Option<&str>, _payload: &P) {} + + /// After a step, or after the pipeline ends (`step` is `None`). + fn after(&self, _step: Option<&str>, _payload: &P) {} + + /// When a Filter fails: for the step, then for the pipeline (`step` is `None`). + fn on_error(&self, _step: Option<&str>, _error: &StepError, _payload: &P) {} +} diff --git a/packages/codeupipe-rs/src/lib.rs b/packages/codeupipe-rs/src/lib.rs new file mode 100644 index 0000000..b7a3f5d --- /dev/null +++ b/packages/codeupipe-rs/src/lib.rs @@ -0,0 +1,112 @@ +//! codeupipe: the eight primitives. +//! +//! Payload, Filter, Predicate, Valve, Tap, Hook, State and Pipeline, as the +//! contract (conformance/API.md) defines them. A Valve is added with +//! `pipeline.add_valve(predicate, filter)`; its type is not a public name. +//! +//! ``` +//! use codeupipe::{Filter, Map, Payload, Pipeline, StepError, Value}; +//! +//! struct GreetFilter; +//! +//! impl Filter for GreetFilter { +//! fn call(&self, payload: &mut Payload) -> Result<(), StepError> { +//! let name = payload.get("name").and_then(Value::as_str).unwrap_or("World").to_string(); +//! payload.insert("greeting", format!("Hello, {name}!")); +//! Ok(()) +//! } +//! } +//! +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_filter(GreetFilter); +//! let out = pipeline.run(&Payload::from_map(Map::from([("name", "Ada")])))?; +//! assert_eq!(out.get("greeting"), Some(&Value::from("Hello, Ada!"))); +//! assert_eq!(pipeline.state().executed(), ["GreetFilter"]); +//! # Ok::<(), StepError>(()) +//! ``` +//! +//! # What the type system refuses +//! +//! The contract refuses a step without the method of its role when it is +//! added. In Rust that is a compile error. +//! +//! A function is not a step: +//! +//! ```compile_fail +//! use codeupipe::{Payload, Pipeline, StepError}; +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_filter(|_payload: &mut Payload| -> Result<(), StepError> { Ok(()) }); +//! ``` +//! +//! The condition of a Valve comes first: +//! +//! ```compile_fail +//! use codeupipe::{Filter, Payload, Pipeline, Predicate, StepError}; +//! struct OnPredicate; +//! impl Predicate for OnPredicate { +//! fn test(&self, _payload: &Payload) -> bool { true } +//! } +//! struct SetFilter; +//! impl Filter for SetFilter { +//! fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { Ok(()) } +//! } +//! let mut pipeline = Pipeline::new(); +//! pipeline.add_valve(SetFilter, OnPredicate); +//! ``` +//! +//! A pipeline carries one type. A Filter for a type of your own +//! (`PipelineData`) does not go into a pipeline of `Payload`: +//! +//! ```compile_fail +//! use codeupipe::{Filter, Payload, Pipeline, PipelineData, StepError}; +//! #[derive(Clone)] +//! struct Order { price: f64 } +//! impl PipelineData for Order {} +//! struct DiscountFilter; +//! impl Filter for DiscountFilter { +//! fn call(&self, order: &mut Order) -> Result<(), StepError> { order.price *= 0.9; Ok(()) } +//! } +//! let mut pipeline: Pipeline = Pipeline::new(); +//! pipeline.add_filter(DiscountFilter); +//! ``` +//! +//! Only a Filter writes. A Tap gets `&Payload`: +//! +//! ```compile_fail +//! use codeupipe::{Payload, StepError, Tap}; +//! struct WritingTap; +//! impl Tap for WritingTap { +//! fn observe(&self, payload: &Payload) -> Result<(), StepError> { +//! payload.insert("seen", true); +//! Ok(()) +//! } +//! } +//! ``` + +mod filter; +mod hook; +mod map; +mod named_pipeline; +mod payload; +mod pipeline; +mod pipeline_data; +mod predicate; +mod state; +mod step_error; +mod step_label; +mod tap; +mod value; +mod valve; + +pub use filter::Filter; +pub use hook::Hook; +pub use map::Map; +pub use named_pipeline::NamedPipeline; +pub use payload::Payload; +pub use pipeline::Pipeline; +pub use pipeline_data::PipelineData; +pub use predicate::Predicate; +pub use state::{SharedError, State}; +pub use step_error::StepError; +pub use tap::Tap; +pub use value::Value; diff --git a/packages/codeupipe-rs/src/map.rs b/packages/codeupipe-rs/src/map.rs new file mode 100644 index 0000000..7573cd7 --- /dev/null +++ b/packages/codeupipe-rs/src/map.rs @@ -0,0 +1,132 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use crate::value::Value; + +/// A map with more keys than this has an index; a smaller one is searched in +/// order, which is faster than hashing for a few keys. +const INDEX_FROM: usize = 8; + +/// A map from text keys to values. Keys keep the order in which they were +/// first inserted, as a Python dict and a JavaScript Map do. +/// +/// Two maps are equal when they have the same entries, in any order, as two +/// JSON objects are. +#[derive(Clone, Default)] +pub struct Map { + // A key is made once and shared, by the index and by a clone of the map. + entries: Vec<(Arc, Value)>, + index: Option, usize>>, +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Map { + pub fn new() -> Self { + Map::default() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn position(&self, key: &str) -> Option { + match &self.index { + Some(index) => index.get(key).copied(), + None => self.entries.iter().position(|(known, _)| &**known == key), + } + } + + pub fn contains_key(&self, key: &str) -> bool { + self.position(key).is_some() + } + + pub fn get(&self, key: &str) -> Option<&Value> { + self.position(key).map(|at| &self.entries[at].1) + } + + /// Set `key` to `value`. A key that is there keeps its place. + pub fn insert(&mut self, key: impl AsRef, value: impl Into) { + let value = value.into(); + let key = key.as_ref(); + if let Some(at) = self.position(key) { + self.entries[at].1 = value; + return; + } + let key: Arc = Arc::from(key); + let at = self.entries.len(); + self.entries.push((Arc::clone(&key), value)); + match &mut self.index { + Some(index) => { + index.insert(key, at); + } + None if self.entries.len() > INDEX_FROM => { + // Room for a few times the size, so that a map that is filled + // key by key does not grow its index at every doubling. + let mut index = HashMap::with_capacity(4 * INDEX_FROM); + for (at, (key, _)) in self.entries.iter().enumerate() { + index.insert(Arc::clone(key), at); + } + self.index = Some(index); + } + None => {} + } + } + + /// Remove `key`, and give its value. Nothing happens if it is absent. + pub fn remove(&mut self, key: &str) -> Option { + let at = self.position(key)?; + let (_, value) = self.entries.remove(at); + if let Some(index) = &mut self.index { + index.remove(key); + for (moved, (key, _)) in self.entries.iter().enumerate().skip(at) { + index.insert(Arc::clone(key), moved); + } + } + Some(value) + } + + pub fn keys(&self) -> impl Iterator { + self.entries.iter().map(|(key, _)| &**key) + } + + pub fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(key, value)| (&**key, value)) + } +} + +impl PartialEq for Map { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() + && self + .iter() + .all(|(key, value)| other.get(key) == Some(value)) + } +} + +impl fmt::Debug for Map { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map().entries(self.iter()).finish() + } +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl, V: Into> FromIterator<(K, V)> for Map { + fn from_iter>(entries: I) -> Self { + let mut map = Map::new(); + for (key, value) in entries { + map.insert(key, value); + } + map + } +} + +impl, V: Into, const N: usize> From<[(K, V); N]> for Map { + fn from(entries: [(K, V); N]) -> Self { + entries.into_iter().collect() + } +} diff --git a/packages/codeupipe-rs/src/named_pipeline.rs b/packages/codeupipe-rs/src/named_pipeline.rs new file mode 100644 index 0000000..5e93539 --- /dev/null +++ b/packages/codeupipe-rs/src/named_pipeline.rs @@ -0,0 +1,12 @@ +use crate::payload::Payload; +use crate::pipeline::Pipeline; + +/// A pipeline with a name: a type that adds the steps. +/// +/// `Pipeline::named(UserOnboarding)` makes the pipeline, and its label is +/// `UserOnboarding`. Rust has no class to extend, so this trait is the named +/// type of the contract (`class UserOnboarding(Pipeline)` in Python). +/// `add_steps` only adds steps. +pub trait NamedPipeline

{ + fn add_steps(&self, pipeline: &mut Pipeline

); +} diff --git a/packages/codeupipe-rs/src/payload.rs b/packages/codeupipe-rs/src/payload.rs new file mode 100644 index 0000000..3cb00be --- /dev/null +++ b/packages/codeupipe-rs/src/payload.rs @@ -0,0 +1,103 @@ +use crate::map::Map; +use crate::pipeline_data::PipelineData; +use crate::value::Value; + +/// Payload: the data that moves through a pipeline. +/// +/// One keyed container. Every write changes it in place and returns it, so +/// writes can be chained. `copy()` gives a separate Payload when you want to +/// keep the one you have. +#[derive(Clone, Debug, Default)] +pub struct Payload { + data: Map, + trace_id: Option, + lineage: Vec, +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Payload { + /// An empty Payload. + pub fn new() -> Self { + Payload::default() + } + + /// A Payload with the contents of `data`. The map is moved in, so the + /// caller's map cannot be changed through the Payload. + pub fn from_map(data: Map) -> Self { + Payload { + data, + ..Payload::default() + } + } + + /// The value for `key`, or absent. A default is `.unwrap_or(...)`. + pub fn get(&self, key: &str) -> Option<&Value> { + self.data.get(key) + } + + /// Whether `key` is present (also when its value is null). + pub fn has(&self, key: &str) -> bool { + self.data.contains_key(key) + } + + /// Set `key` to `value` in this Payload, and return it. + pub fn insert + Into>( + &mut self, + key: K, + value: impl Into, + ) -> &mut Self { + self.data.insert(key, value); + self + } + + /// Remove `key` from this Payload (nothing happens if it is absent), and return it. + pub fn remove(&mut self, key: &str) -> &mut Self { + self.data.remove(key); + self + } + + /// Write every key of `other` into this Payload (`other` wins), and return it. + pub fn merge(&mut self, other: &Payload) -> &mut Self { + for (key, value) in other.data.iter() { + self.data.insert(key, value.clone()); + } + if self.trace_id.is_none() { + self.trace_id = other.trace_id.clone(); + } + self.lineage.extend(other.lineage.iter().cloned()); + self + } + + /// A separate Payload with the same contents, trace id and lineage. + pub fn copy(&self) -> Payload { + self.clone() + } + + /// The contents as a new map. + pub fn to_dict(&self) -> Map { + self.data.clone() + } + + /// Set the trace id on this Payload, and return it. + pub fn with_trace(&mut self, trace_id: impl Into) -> &mut Self { + self.trace_id = Some(trace_id.into()); + self + } + + /// The trace id, for following one Payload across systems. + pub fn trace_id(&self) -> Option<&str> { + self.trace_id.as_deref() + } + + /// The steps this Payload has passed through, when lineage is on. + pub fn lineage(&self) -> &[String] { + &self.lineage + } +} + +impl PipelineData for Payload { + /// Lineage: the label of each step, in `lineage()`. + fn stamp(&mut self, label: &str) { + self.lineage.push(label.to_string()); + } +} diff --git a/packages/codeupipe-rs/src/pipeline.rs b/packages/codeupipe-rs/src/pipeline.rs new file mode 100644 index 0000000..a99c2d4 --- /dev/null +++ b/packages/codeupipe-rs/src/pipeline.rs @@ -0,0 +1,326 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::fmt; +use std::rc::Rc; +use std::time::Instant; + +use crate::filter::Filter; +use crate::hook::Hook; +use crate::named_pipeline::NamedPipeline; +use crate::payload::Payload; +use crate::pipeline_data::PipelineData; +use crate::predicate::Predicate; +use crate::state::State; +use crate::step_error::StepError; +use crate::step_label::label_of; +use crate::tap::Tap; +use crate::valve::Valve; + +/// A step that Hooks see: it runs, and its failure stops the run. +enum Gated

{ + Filter(Box>), + Valve(Valve

), + Pipeline(Box>), +} + +enum Step

{ + Tap(Box>), + Gated(Gated

), +} + +struct Labeled

{ + label: String, + step: Step

, +} + +/// Pipeline: runs the steps in order. +/// +/// Filters change the Payload, Valves gate a Filter, Taps read, Hooks see the +/// lifecycle, and State records what ran. Add steps with `add_filter`, +/// `add_valve`, `add_tap` and `add_pipeline`, add Hooks with `use_hook`, and +/// run with `pipeline.run(&payload)`. +/// +/// It carries a `Payload` unless you give it a type of your own +/// (`Pipeline`, see `PipelineData`). +pub struct Pipeline

{ + label: String, + steps: Vec>, + uses: HashMap, + hooks: Vec>>, + runs: Runs, + timing: bool, + lineage: bool, +} + +/// The runs of a pipeline. Each run writes into a State of its own; this +/// keeps the State of the run that started last, for `state()`. +#[derive(Default)] +struct Runs { + /// The number of runs that have started; a run's number is its ticket. + started: Cell, + /// The State of the run that started last, and that has ended. + last: RefCell>, + /// A State that no run and no reader has, kept for its storage. + spare: RefCell>, +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Runs { + fn start(&self) -> u64 { + let ticket = self.started.get() + 1; + self.started.set(ticket); + ticket + } + + /// A State for a run, with the storage of an earlier one when there is one. + fn new_state(&self) -> State { + let mut state = self.spare.take().unwrap_or_default(); + state.reset(); + state + } + + /// Keep `state` as the State of the pipeline, unless a run that started + /// later has ended already (a run inside this run). + fn end(&self, ticket: u64, state: State) { + if ticket != self.started.get() { + self.spare.replace(Some(state)); + return; + } + let earlier = self.last.replace(Rc::new(state)); + if let Ok(earlier) = Rc::try_unwrap(earlier) { + self.spare.replace(Some(earlier)); + } + } +} + +impl Default for Pipeline

{ + fn default() -> Self { + Pipeline::new() + } +} + +impl

fmt::Debug for Pipeline

{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pipeline") + .field("label", &self.label) + .field("steps", &self.labels()) + .finish() + } +} + +impl

Pipeline

{ + /// The label of each step, in order. + pub fn labels(&self) -> Vec { + self.steps.iter().map(|step| step.label.clone()).collect() + } +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Pipeline

{ + /// A pipeline with no name. Nested, its label is `Pipeline`. + pub fn new() -> Self { + Pipeline { + label: "Pipeline".to_string(), + steps: Vec::new(), + uses: HashMap::new(), + hooks: Vec::new(), + runs: Runs::default(), + timing: false, + lineage: false, + } + } + + /// The pipeline that `named` adds the steps of. Its label is the name of + /// the type of `named`. + pub fn named>(named: N) -> Self { + let mut pipeline = Pipeline::new(); + pipeline.label = label_of::(); + named.add_steps(&mut pipeline); + pipeline + } + + /// The State of the run that started last, once it has ended. + /// + /// A State belongs to one run, and this is for a caller with one run at a + /// time. It is a snapshot: a later run does not change it. A caller with + /// more than one run at a time gives each run its own State + /// (`run_with_state`). + pub fn state(&self) -> Rc { + Rc::clone(&self.runs.last.borrow()) + } + + /// The type name of the step, with a number from its second use in this pipeline. + fn next_label(&mut self, base: String) -> String { + let count = self.uses.entry(base.clone()).or_insert(0); + *count += 1; + if *count == 1 { + base + } else { + format!("{base}#{count}") + } + } + + fn push(&mut self, base: String, step: Step

) { + let label = self.next_label(base); + self.steps.push(Labeled { label, step }); + } + + /// Add a Filter. + pub fn add_filter + 'static>(&mut self, filter: F) { + self.push( + label_of::(), + Step::Gated(Gated::Filter(Box::new(filter))), + ); + } + + /// Add a Valve: `filter` runs only when `predicate.test(payload)` is true. + /// + /// The condition comes first, then the Filter that it controls. When the + /// test is false the Payload passes through unchanged and the step is + /// recorded as skipped. The step has the label of its Filter. + pub fn add_valve + 'static, F: Filter

+ 'static>( + &mut self, + predicate: Q, + filter: F, + ) { + let valve = Valve { + predicate: Box::new(predicate), + filter: Box::new(filter), + }; + self.push(label_of::(), Step::Gated(Gated::Valve(valve))); + } + + /// Add a Tap. + pub fn add_tap + 'static>(&mut self, tap: T) { + self.push(label_of::(), Step::Tap(Box::new(tap))); + } + + /// Nest a Pipeline as one step of this Pipeline. + pub fn add_pipeline(&mut self, pipeline: Pipeline

) { + self.push( + pipeline.label.clone(), + Step::Gated(Gated::Pipeline(Box::new(pipeline))), + ); + } + + /// Add a Hook. + pub fn use_hook + 'static>(&mut self, hook: H) { + self.hooks.push(Box::new(hook)); + } + + /// Turn timing (the time of each step, in State) and lineage (labels on + /// the Payload) on or off. Both are off by default. + pub fn observe(&mut self, timing: bool, lineage: bool) { + self.timing = timing; + self.lineage = lineage; + } + + /// Run the steps in order on a copy of `input`, and return the result. + /// + /// A Filter failure stops the run and is returned after the Hooks have + /// seen it. A Tap failure is recorded in State and the run continues. + /// The run writes into a new State, which `state()` gives afterwards. + /// + /// It takes `&self`: a step can hold the pipeline that it is in (an `Rc`) + /// and run it again, and each run has its own State. + pub fn run(&self, input: &P) -> Result { + let ticket = self.runs.start(); + let mut state = self.runs.new_state(); + let mut payload = input.clone(); // the caller's value is not changed + let result = self.run_steps(&mut payload, &mut state); + self.runs.end(ticket, state); + result.map(|()| payload) + } + + /// `run`, writing into `state`, which the caller gives. A caller with more + /// than one run at a time gives each run its own State. + pub fn run_with_state(&self, input: &P, state: &mut State) -> Result { + let ticket = self.runs.start(); + let mut payload = input.clone(); // the caller's value is not changed + let result = self.run_steps(&mut payload, state); + self.runs.end(ticket, state.clone()); + result.map(|()| payload) + } + + fn run_steps(&self, payload: &mut P, state: &mut State) -> Result<(), StepError> { + let Pipeline { + steps, + hooks, + timing, + lineage, + .. + } = self; + + for hook in hooks.iter() { + hook.before(None, payload); + } + + for Labeled { label, step } in steps.iter() { + let gated = match step { + Step::Tap(tap) => { + if let Err(error) = tap.observe(payload) { + state.record_error(label, error); + } + state.mark_executed(label); + continue; + } + Step::Gated(gated) => gated, + }; + + let started = if *timing { Some(Instant::now()) } else { None }; + for hook in hooks.iter() { + hook.before(Some(label), payload); + } + + match run_gated(gated, payload) { + Ok(true) => state.mark_executed(label), + Ok(false) => state.mark_skipped(label), + Err(error) => { + if let Some(started) = started { + state.record_timing(label, started.elapsed()); + } + // The failing step first, then the pipeline: on_error + // fires at both levels, as before and after do. + for hook in hooks.iter() { + hook.on_error(Some(label), &error, payload); + } + for hook in hooks.iter() { + hook.on_error(None, &error, payload); + } + return Err(error); + } + } + + for hook in hooks.iter() { + hook.after(Some(label), payload); + } + if let Some(started) = started { + state.record_timing(label, started.elapsed()); + } + if *lineage { + payload.stamp(label); + } + } + + for hook in hooks.iter() { + hook.after(None, payload); + } + Ok(()) + } +} + +/// Run one step that Hooks see. `Ok(false)` is a Valve that skipped its Filter. +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +fn run_gated(gated: &Gated

, payload: &mut P) -> Result { + match gated { + Gated::Filter(filter) => filter.call(payload).map(|()| true), + Gated::Valve(valve) if !valve.predicate.test(payload) => Ok(false), + Gated::Valve(valve) => valve.filter.call(payload).map(|()| true), + Gated::Pipeline(inner) => { + // A run of its own, with its own State: a failure inside leaves this + // Payload as it was. + *payload = inner.run(payload)?; + Ok(true) + } + } +} diff --git a/packages/codeupipe-rs/src/pipeline_data.rs b/packages/codeupipe-rs/src/pipeline_data.rs new file mode 100644 index 0000000..75ac028 --- /dev/null +++ b/packages/codeupipe-rs/src/pipeline_data.rs @@ -0,0 +1,39 @@ +/// What a Pipeline carries from step to step. +/// +/// `Payload` is the default: a map of JSON-shaped values that any developer +/// can read, and that every runtime of codeupipe has. A type of your own can +/// take its place, such as a struct with the fields that your steps use: +/// +/// ``` +/// use codeupipe::{Filter, Pipeline, PipelineData, StepError}; +/// +/// #[derive(Clone)] +/// struct Order { price: f64 } +/// +/// impl PipelineData for Order {} +/// +/// struct DiscountFilter; +/// +/// impl Filter for DiscountFilter { +/// fn call(&self, order: &mut Order) -> Result<(), StepError> { +/// order.price *= 0.9; +/// Ok(()) +/// } +/// } +/// +/// let mut pipeline = Pipeline::new(); +/// pipeline.add_filter(DiscountFilter); +/// assert_eq!(pipeline.run(&Order { price: 100.0 })?.price, 90.0); +/// # Ok::<(), StepError>(()) +/// ``` +/// +/// The Filters, Predicates, Taps and Hooks of that pipeline then take the +/// type, and a step for another type is a compile error. The rules stay the +/// same: `run` works on a clone of the input, only a Filter writes, a Tap +/// failure is recorded and the run goes on. +pub trait PipelineData: Clone + 'static { + /// Called after each step when lineage is on (`observe(_, true)`), with + /// the label of the step. `Payload` keeps the labels; a type of your own + /// keeps them only if it implements this. + fn stamp(&mut self, _label: &str) {} +} diff --git a/packages/codeupipe-rs/src/predicate.rs b/packages/codeupipe-rs/src/predicate.rs new file mode 100644 index 0000000..4a5c43f --- /dev/null +++ b/packages/codeupipe-rs/src/predicate.rs @@ -0,0 +1,9 @@ +use crate::payload::Payload; + +/// Predicate: asks a yes-or-no question about the data. +/// +/// A Predicate is a type. Its settings (a threshold, a flag) are on the +/// instance. It reads the Payload and does not write to it. +pub trait Predicate

{ + fn test(&self, payload: &P) -> bool; +} diff --git a/packages/codeupipe-rs/src/state.rs b/packages/codeupipe-rs/src/state.rs new file mode 100644 index 0000000..4dbb247 --- /dev/null +++ b/packages/codeupipe-rs/src/state.rs @@ -0,0 +1,115 @@ +use std::error::Error; +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use crate::step_error::StepError; + +/// An error that State keeps. It is shared, so that a State can be cloned. +pub type SharedError = Arc; + +/// State: what happened in one run. +/// +/// Which steps executed, which a Valve skipped, the errors, and the timings +/// when timing is on. A State belongs to one run: give one to +/// `pipeline.run_with_state`, or read the State of the run that started last +/// with `pipeline.state()`. +#[derive(Clone, Debug, Default)] +pub struct State { + executed: Labels, + skipped: Labels, + errors: Vec<(String, SharedError)>, + timings: Vec<(String, Duration)>, +} + +/// A list of labels whose Strings are kept from one run to the next and +/// written over, so that a run like the last one makes no new String. +#[derive(Clone, Default)] +struct Labels { + all: Vec, + len: usize, +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Labels { + fn push(&mut self, label: &str) { + match self.all.get_mut(self.len) { + Some(slot) => { + slot.clear(); + slot.push_str(label); + } + None => self.all.push(label.to_string()), + } + self.len += 1; + } + + fn as_slice(&self) -> &[String] { + &self.all[..self.len] + } + + fn clear(&mut self) { + self.len = 0; + } +} + +impl fmt::Debug for Labels { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.as_slice()).finish() + } +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl State { + /// An empty State, for a caller that gives the State of a run + /// (`pipeline.run_with_state(&input, &mut state)`). + pub fn new() -> Self { + State::default() + } + + /// Step labels, in order. + pub fn executed(&self) -> &[String] { + self.executed.as_slice() + } + + /// Steps that a Valve did not run. + pub fn skipped(&self) -> &[String] { + self.skipped.as_slice() + } + + /// Pairs of label and error. + pub fn errors(&self) -> &[(String, SharedError)] { + &self.errors + } + + /// Label and time, in order, when timing is on. + pub fn timings(&self) -> &[(String, Duration)] { + &self.timings + } + + /// Empty, for a new run. The storage is kept. + pub(crate) fn reset(&mut self) { + self.executed.clear(); + self.skipped.clear(); + self.errors.clear(); + self.timings.clear(); + } + + pub(crate) fn mark_executed(&mut self, label: &str) { + self.executed.push(label); + } + + pub(crate) fn mark_skipped(&mut self, label: &str) { + self.skipped.push(label); + } + + pub(crate) fn record_error(&mut self, label: &str, error: StepError) { + self.errors.push((label.to_string(), Arc::from(error))); + } + + pub(crate) fn record_timing(&mut self, label: &str, time: Duration) { + match self.timings.iter_mut().find(|(known, _)| known == label) { + Some(entry) => entry.1 = time, + None => self.timings.push((label.to_string(), time)), + } + } +} diff --git a/packages/codeupipe-rs/src/step_error.rs b/packages/codeupipe-rs/src/step_error.rs new file mode 100644 index 0000000..a36d19a --- /dev/null +++ b/packages/codeupipe-rs/src/step_error.rs @@ -0,0 +1,3 @@ +/// The error of a step that fails. Any error type converts into it, and so +/// does text: `Err("sink is down".into())`. +pub type StepError = Box; diff --git a/packages/codeupipe-rs/src/step_label.rs b/packages/codeupipe-rs/src/step_label.rs new file mode 100644 index 0000000..030c3d7 --- /dev/null +++ b/packages/codeupipe-rs/src/step_label.rs @@ -0,0 +1,7 @@ +/// The label of a step: the name of its type, without its path and without +/// generic arguments. `my_app::pricing::DiscountFilter` gives `DiscountFilter`. +pub(crate) fn label_of() -> String { + let full = std::any::type_name::(); + let path = full.split('<').next().unwrap_or(full); + path.rsplit("::").next().unwrap_or(path).to_string() +} diff --git a/packages/codeupipe-rs/src/tap.rs b/packages/codeupipe-rs/src/tap.rs new file mode 100644 index 0000000..3a4f9f5 --- /dev/null +++ b/packages/codeupipe-rs/src/tap.rs @@ -0,0 +1,8 @@ +use crate::payload::Payload; +use crate::step_error::StepError; + +/// Tap: reads the data as it goes by. It does not write to the Payload. +/// A Tap failure is recorded in State and never stops the run. +pub trait Tap

{ + fn observe(&self, payload: &P) -> Result<(), StepError>; +} diff --git a/packages/codeupipe-rs/src/value.rs b/packages/codeupipe-rs/src/value.rs new file mode 100644 index 0000000..8b1667d --- /dev/null +++ b/packages/codeupipe-rs/src/value.rs @@ -0,0 +1,125 @@ +use std::fmt; + +use crate::map::Map; + +/// A JSON-shaped value: what a Payload holds under a key. +/// +/// There is one number type, as in JSON and JavaScript: `1` and `1.0` are +/// the same value. `true` is never `1`. +#[derive(Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Number(f64), + String(String), + List(Vec), + Map(Map), +} + +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl Value { + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(value) => Some(*value), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Value::Number(value) => Some(*value), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(value) => Some(value), + _ => None, + } + } + + pub fn as_list(&self) -> Option<&[Value]> { + match self { + Value::List(value) => Some(value), + _ => None, + } + } + + pub fn as_map(&self) -> Option<&Map> { + match self { + Value::Map(value) => Some(value), + _ => None, + } + } +} + +/// Written as JSON is, so that a printed Payload reads like the data. +// cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Value::Null => f.write_str("null"), + Value::Bool(value) => write!(f, "{value}"), + Value::Number(value) if value.fract() == 0.0 && value.abs() < 1e15 => { + write!(f, "{}", *value as i64) + } + Value::Number(value) => write!(f, "{value}"), + Value::String(value) => write!(f, "{value:?}"), + Value::List(values) => f.debug_list().entries(values).finish(), + Value::Map(map) => write!(f, "{map:?}"), + } + } +} + +impl From for Value { + fn from(value: bool) -> Self { + Value::Bool(value) + } +} + +macro_rules! number_from { + ($($t:ty),*) => { + $(impl From<$t> for Value { + fn from(value: $t) -> Self { + Value::Number(value as f64) + } + })* + }; +} + +number_from!(f64, f32, i8, i16, i32, i64, isize, u8, u16, u32, u64, usize); + +impl From<&str> for Value { + fn from(value: &str) -> Self { + Value::String(value.to_string()) + } +} + +impl From for Value { + fn from(value: String) -> Self { + Value::String(value) + } +} + +impl From> for Value { + fn from(value: Vec) -> Self { + Value::List(value) + } +} + +impl From for Value { + fn from(value: Map) -> Self { + Value::Map(value) + } +} + +impl> From> for Value { + fn from(value: Option) -> Self { + value.map_or(Value::Null, Into::into) + } +} diff --git a/packages/codeupipe-rs/src/valve.rs b/packages/codeupipe-rs/src/valve.rs new file mode 100644 index 0000000..4bc327e --- /dev/null +++ b/packages/codeupipe-rs/src/valve.rs @@ -0,0 +1,11 @@ +use crate::filter::Filter; +use crate::predicate::Predicate; + +/// Valve: joins a Predicate to a Filter. The Filter runs only when the +/// Predicate is true. +/// +/// `Pipeline::add_valve(predicate, filter)` makes one. It is not a public name. +pub(crate) struct Valve

{ + pub(crate) predicate: Box>, + pub(crate) filter: Box>, +} diff --git a/packages/codeupipe-rs/tests/map.rs b/packages/codeupipe-rs/tests/map.rs new file mode 100644 index 0000000..a57eb87 --- /dev/null +++ b/packages/codeupipe-rs/tests/map.rs @@ -0,0 +1,90 @@ +use codeupipe::{Map, Value}; + +/// A map with `n` keys k0, k1, ... in order, each with its number. +fn numbered(n: usize) -> Map { + (0..n).map(|i| (format!("k{i}"), i)).collect() +} + +// A small map is searched in order and a large one through an index, so each +// test runs on both sides of the size where the index starts. +const SIZES: [usize; 4] = [3, 8, 9, 40]; + +#[test] +fn get_finds_every_key() { + for size in SIZES { + let map = numbered(size); + for i in 0..size { + assert_eq!(map.get(&format!("k{i}")), Some(&Value::from(i)), "{size}"); + } + assert_eq!(map.get("absent"), None); + assert!(!map.contains_key("absent")); + } +} + +#[test] +fn an_insert_of_a_key_that_is_there_keeps_its_place() { + for size in SIZES { + let mut map = numbered(size); + map.insert("k1", "new"); + assert_eq!(map.len(), size); + assert_eq!(map.keys().nth(1), Some("k1")); + assert_eq!(map.get("k1"), Some(&Value::from("new"))); + } +} + +#[test] +fn a_remove_keeps_the_order_and_every_other_key() { + for size in SIZES { + let mut map = numbered(size); + assert_eq!(map.remove("k1"), Some(Value::from(1))); + assert_eq!(map.remove("k1"), None); + assert_eq!(map.len(), size - 1); + let want: Vec = (0..size) + .filter(|&i| i != 1) + .map(|i| format!("k{i}")) + .collect(); + assert_eq!(map.keys().collect::>(), want); + for i in (0..size).filter(|&i| i != 1) { + assert_eq!(map.get(&format!("k{i}")), Some(&Value::from(i)), "{size}"); + } + map.insert("k1", "back"); + assert_eq!(map.keys().last(), Some("k1")); + assert_eq!(map.get("k1"), Some(&Value::from("back"))); + } +} + +#[test] +fn a_map_that_grows_past_the_index_and_shrinks_back_still_finds_its_keys() { + let mut map = numbered(20); + for i in 0..18 { + map.remove(&format!("k{i}")); + } + assert_eq!(map.keys().collect::>(), ["k18", "k19"]); + assert_eq!(map.get("k19"), Some(&Value::from(19))); + map.insert("k0", 0); + assert_eq!(map.get("k0"), Some(&Value::from(0))); +} + +#[test] +fn a_clone_is_separate() { + for size in SIZES { + let map = numbered(size); + let mut other = map.clone(); + other.insert("k0", "changed"); + other.insert("extra", true); + assert_eq!(map.get("k0"), Some(&Value::from(0))); + assert!(!map.contains_key("extra")); + } +} + +#[test] +fn two_maps_with_the_same_entries_in_another_order_are_equal() { + for size in SIZES { + let forward = numbered(size); + let backward: Map = (0..size).rev().map(|i| (format!("k{i}"), i)).collect(); + assert_eq!(forward, backward); + let mut different = numbered(size); + different.insert("k0", "x"); + assert_ne!(forward, different); + } +} diff --git a/packages/codeupipe-rs/tests/payload.rs b/packages/codeupipe-rs/tests/payload.rs new file mode 100644 index 0000000..4e90e4b --- /dev/null +++ b/packages/codeupipe-rs/tests/payload.rs @@ -0,0 +1,168 @@ +use codeupipe::{Map, Payload, Value}; + +#[test] +fn the_map_that_the_caller_gives_is_copied() { + let data = Map::from([("a", 1)]); + let mut payload = Payload::from_map(data.clone()); + payload.insert("b", 2); + assert_eq!(data, Map::from([("a", 1)])); + assert_eq!(payload.to_dict(), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn an_empty_payload_has_no_keys() { + assert_eq!(Payload::new().to_dict(), Map::new()); +} + +#[test] +fn get_gives_the_value_or_absent() { + let payload = Payload::from_map(Map::from([("a", Value::from(1)), ("n", Value::Null)])); + assert_eq!(payload.get("a"), Some(&Value::from(1))); + assert_eq!(payload.get("missing"), None); + // A default is the language's own: unwrap_or. A present null is not absent. + assert_eq!( + payload.get("missing").cloned().unwrap_or(Value::from(7)), + Value::from(7) + ); + assert_eq!( + payload.get("n").cloned().unwrap_or(Value::from(7)), + Value::Null + ); +} + +#[test] +fn has_is_true_for_a_key_with_a_null_value() { + let payload = Payload::from_map(Map::from([("n", Value::Null)])); + assert!(payload.has("n")); + assert!(!payload.has("missing")); +} + +#[test] +fn insert_remove_merge_and_with_trace_change_the_payload_and_return_it() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + let other = Payload::from_map(Map::from([("c", 3)])); + let returned: *const Payload = payload.insert("b", 2); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.remove("a"); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.merge(&other); + assert!(std::ptr::eq(returned, &payload)); + let returned: *const Payload = payload.with_trace("t-1"); + assert!(std::ptr::eq(returned, &payload)); + assert_eq!(payload.to_dict(), Map::from([("b", 2), ("c", 3)])); + assert_eq!(payload.trace_id(), Some("t-1")); +} + +#[test] +fn writes_chain() { + let mut payload = Payload::new(); + payload.insert("a", 1).insert("b", 2).remove("a"); + assert_eq!(payload.to_dict(), Map::from([("b", 2)])); +} + +#[test] +fn remove_of_a_key_that_is_not_there_does_nothing() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + payload.remove("zz"); + assert_eq!(payload.to_dict(), Map::from([("a", 1)])); +} + +#[test] +fn in_a_merge_the_other_payload_wins() { + let mut payload = Payload::from_map(Map::from([("a", 1), ("b", 1)])); + payload.merge(&Payload::from_map(Map::from([("b", 2)]))); + assert_eq!(payload.to_dict(), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn a_merge_keeps_the_trace_id_it_has_and_takes_one_when_it_has_none() { + let mut other = Payload::new(); + other.with_trace("other"); + + let mut mine = Payload::new(); + mine.with_trace("mine").merge(&other); + assert_eq!(mine.trace_id(), Some("mine")); + + let mut none = Payload::new(); + none.merge(&other); + assert_eq!(none.trace_id(), Some("other")); +} + +#[test] +fn copy_gives_a_separate_payload_with_the_same_contents() { + let mut payload = Payload::from_map(Map::from([("a", 1)])); + payload.with_trace("t-1"); + let mut copy = payload.copy(); + copy.insert("a", 2); + assert_eq!(payload.get("a"), Some(&Value::from(1))); + assert_eq!(copy.trace_id(), Some("t-1")); +} + +#[test] +fn a_nested_value_changed_in_a_copy_is_not_changed_in_the_original() { + // D6: copy() is deep in Rust. A shared value could not be changed in + // place anyway, so the behaviour is the same as a shallow copy. + let payload = Payload::from_map(Map::from([("list", Value::List(vec![Value::from(1)]))])); + let mut copy = payload.copy(); + let mut list = copy + .get("list") + .and_then(Value::as_list) + .unwrap_or(&[]) + .to_vec(); + list.push(Value::from(2)); + copy.insert("list", list); + assert_eq!( + payload.get("list"), + Some(&Value::List(vec![Value::from(1)])) + ); +} + +#[test] +fn to_dict_gives_a_new_map_each_time() { + let payload = Payload::from_map(Map::from([("a", 1)])); + let mut out = payload.to_dict(); + out.insert("a", 2); + assert_eq!(payload.get("a"), Some(&Value::from(1))); +} + +#[test] +fn the_trace_id_is_absent_until_it_is_set_and_the_lineage_starts_empty() { + let payload = Payload::new(); + assert_eq!(payload.trace_id(), None); + assert!(payload.lineage().is_empty()); +} + +#[test] +fn keys_keep_the_order_in_which_they_were_inserted() { + let mut payload = Payload::new(); + payload + .insert("b", 1) + .insert("a", 2) + .insert("c", 3) + .remove("a"); + payload.insert("a", 4); + let out = payload.to_dict(); + assert_eq!(out.keys().collect::>(), ["b", "c", "a"]); +} + +#[test] +fn numbers_compare_as_json_numbers_and_true_is_not_one() { + assert_eq!(Value::from(1), Value::from(1.0)); + assert_ne!(Value::from(true), Value::from(1)); +} + +#[test] +fn maps_with_the_same_entries_are_equal_in_any_order() { + assert_eq!( + Map::from([("a", 1), ("b", 2)]), + Map::from([("b", 2), ("a", 1)]) + ); + assert_ne!(Map::from([("a", 1)]), Map::from([("a", 1), ("b", 2)])); +} + +#[test] +fn a_payload_can_go_to_another_thread() { + fn send_and_sync() {} + send_and_sync::(); + send_and_sync::(); +} diff --git a/packages/codeupipe-rs/tests/pipeline.rs b/packages/codeupipe-rs/tests/pipeline.rs new file mode 100644 index 0000000..22ad5bf --- /dev/null +++ b/packages/codeupipe-rs/tests/pipeline.rs @@ -0,0 +1,464 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{ + Filter, Hook, Map, NamedPipeline, Payload, Pipeline, Predicate, StepError, Tap, Value, +}; + +fn n(payload: &Payload) -> f64 { + payload.get("n").and_then(Value::as_f64).unwrap_or(0.0) +} + +struct AddFilter; + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("n", n(payload) + 1.0); + Ok(()) + } +} + +struct AddOtherFilter; + +impl Filter for AddOtherFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("n", n(payload) + 1.0); + Ok(()) + } +} + +struct FailFilter; + +impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err("boom".into()) + } +} + +/// Writes, then fails: a Hook sees the Payload as the step left it. +struct HalfDoneFilter; + +impl Filter for HalfDoneFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("half", true); + Err("stopped half way".into()) + } +} + +struct IsOnPredicate; + +impl Predicate for IsOnPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("on") == Some(&Value::Bool(true)) + } +} + +struct SeeTap { + seen: Rc>>>, +} + +impl Tap for SeeTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + self.seen.borrow_mut().push(payload.get("n").cloned()); + Ok(()) + } +} + +struct FailTap; + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink is down".into()) + } +} + +struct FailOtherTap; + +impl Tap for FailOtherTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink is down again".into()) + } +} + +struct InnerPipeline; + +impl NamedPipeline for InnerPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + } +} + +#[derive(Clone, Default)] +struct RecordHook { + events: Rc>>, +} + +impl RecordHook { + fn events(&self) -> Vec { + self.events.borrow().clone() + } +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("before:{}", step.unwrap_or(""))); + } + fn after(&self, step: Option<&str>, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("after:{}", step.unwrap_or(""))); + } + fn on_error(&self, step: Option<&str>, _error: &StepError, _payload: &Payload) { + self.events + .borrow_mut() + .push(format!("error:{}", step.unwrap_or(""))); + } +} + +fn labels_of_errors(pipeline: &Pipeline) -> Vec { + pipeline + .state() + .errors() + .iter() + .map(|(label, _)| label.clone()) + .collect() +} + +#[test] +fn an_empty_pipeline_returns_the_data_it_was_given() { + let out = Pipeline::new() + .run(&Payload::from_map(Map::from([("a", 1)]))) + .unwrap(); + assert_eq!(out.to_dict(), Map::from([("a", 1)])); +} + +#[test] +fn steps_run_in_order_and_state_records_them() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddOtherFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 2.0); + assert_eq!(pipeline.state().executed(), ["AddFilter", "AddOtherFilter"]); + assert!(pipeline.state().skipped().is_empty()); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn run_works_on_a_copy_so_the_callers_payload_is_not_changed() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + let given = Payload::from_map(Map::from([("n", 1)])); + let out = pipeline.run(&given).unwrap(); + assert_eq!(n(&given), 1.0); + assert_eq!(n(&out), 2.0); +} + +#[test] +fn a_valve_runs_its_filter_when_the_predicate_is_true() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + let out = pipeline + .run(&Payload::from_map(Map::from([("on", true)]))) + .unwrap(); + assert_eq!(n(&out), 1.0); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); + assert!(pipeline.state().skipped().is_empty()); +} + +#[test] +fn a_valve_skips_its_filter_when_the_predicate_is_false() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + let out = pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + assert!(!out.has("n")); + assert!(pipeline.state().executed().is_empty()); + assert_eq!(pipeline.state().skipped(), ["AddFilter"]); +} + +#[test] +fn a_filter_failure_stops_the_run_and_state_keeps_the_steps_before_it() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(FailFilter); + pipeline.add_filter(AddFilter); + let error = pipeline.run(&Payload::new()).unwrap_err(); + assert_eq!(error.to_string(), "boom"); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); +} + +#[test] +fn a_tap_reads_the_payload_and_counts_as_executed() { + let seen = Rc::new(RefCell::new(Vec::new())); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap { seen: seen.clone() }); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!(*seen.borrow(), [Some(Value::from(1))]); + assert_eq!(pipeline.state().executed(), ["AddFilter", "SeeTap"]); +} + +#[test] +fn a_tap_failure_is_recorded_and_does_not_stop_the_run() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_tap(FailTap); + pipeline.add_tap(FailOtherTap); + pipeline.add_filter(AddFilter); + pipeline.use_hook(hook.clone()); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 1.0); + assert_eq!( + pipeline.state().executed(), + ["FailTap", "FailOtherTap", "AddFilter"] + ); + assert_eq!(labels_of_errors(&pipeline), ["FailTap", "FailOtherTap"]); + assert_eq!(pipeline.state().errors()[0].1.to_string(), "sink is down"); + assert_eq!( + hook.events(), + ["before:", "before:AddFilter", "after:AddFilter", "after:"] + ); +} + +#[test] +fn a_nested_pipeline_is_one_step() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); + pipeline.add_filter(AddFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert_eq!(n(&out), 3.0); + assert_eq!(pipeline.state().executed(), ["InnerPipeline", "AddFilter"]); +} + +#[test] +fn hooks_see_the_pipeline_and_each_filter_or_valve_in_order() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_valve(IsOnPredicate, AddOtherFilter); + pipeline.add_tap(SeeTap { + seen: Rc::default(), + }); + pipeline.use_hook(hook.clone()); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!( + hook.events(), + [ + "before:", + "before:AddFilter", + "after:AddFilter", + "before:AddOtherFilter", + "after:AddOtherFilter", + "after:" + ] + ); +} + +#[test] +fn on_a_filter_failure_a_hook_gets_the_step_then_the_pipeline_and_no_after() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(FailFilter); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + assert_eq!( + hook.events(), + [ + "before:", + "before:AddFilter", + "after:AddFilter", + "before:FailFilter", + "error:FailFilter", + "error:" + ] + ); +} + +#[test] +fn a_hook_gets_the_error_itself() { + #[derive(Clone, Default)] + struct KeepErrorHook { + seen: Rc>>, + } + impl Hook for KeepErrorHook { + fn on_error(&self, _step: Option<&str>, error: &StepError, _payload: &Payload) { + self.seen.borrow_mut().push(address(error)); + } + } + fn address(error: &StepError) -> usize { + &**error as *const (dyn std::error::Error + Send + Sync) as *const () as usize + } + + let hook = KeepErrorHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(FailFilter); + pipeline.use_hook(hook.clone()); + let error = pipeline.run(&Payload::new()).unwrap_err(); + // Both calls get the error that run returns: the same value, not a copy. + assert_eq!(*hook.seen.borrow(), [address(&error), address(&error)]); +} + +#[test] +fn on_error_gets_the_payload_as_the_failing_step_left_it() { + #[derive(Clone, Default)] + struct KeepPayloadHook { + seen: Rc>>, + } + impl Hook for KeepPayloadHook { + fn on_error(&self, _step: Option<&str>, _error: &StepError, payload: &Payload) { + self.seen.borrow_mut().push(payload.to_dict()); + } + } + + let hook = KeepPayloadHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(HalfDoneFilter); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + let expected = Map::from([("n", Value::from(1)), ("half", Value::from(true))]); + assert_eq!(*hook.seen.borrow(), [expected.clone(), expected]); +} + +#[test] +fn timing_and_lineage_are_off_by_default() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + let out = pipeline.run(&Payload::new()).unwrap(); + assert!(pipeline.state().timings().is_empty()); + assert!(out.lineage().is_empty()); +} + +#[test] +fn observe_turns_timing_on_and_lineage_when_asked() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap { + seen: Rc::default(), + }); + pipeline.add_valve(IsOnPredicate, AddOtherFilter); + pipeline.observe(true, true); + let out = pipeline.run(&Payload::new()).unwrap(); + let state = pipeline.state(); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["AddFilter", "AddOtherFilter"]); + assert_eq!(out.lineage(), ["AddFilter", "AddOtherFilter"]); +} + +#[test] +fn each_run_starts_with_a_new_state() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsOnPredicate, AddFilter); + pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + pipeline + .run(&Payload::from_map(Map::from([("on", true)]))) + .unwrap(); + assert_eq!(pipeline.state().executed(), ["AddFilter"]); + assert!(pipeline.state().skipped().is_empty()); +} + +struct FailIfAskedFilter; +impl Filter for FailIfAskedFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("fail") == Some(&Value::Bool(true)) { + return Err("asked to fail".into()); + } + Ok(()) + } +} +struct FailIfAskedTap; +impl Tap for FailIfAskedTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + if payload.get("tap_fail") == Some(&Value::Bool(true)) { + return Err("asked to fail".into()); + } + Ok(()) + } +} + +#[test] +fn a_shorter_run_after_a_longer_one_keeps_nothing_of_it() { + let mut pipeline = Pipeline::new(); + pipeline.observe(true, false); + pipeline.add_filter(FailIfAskedFilter); + pipeline.add_valve(IsOnPredicate, AddFilter); + pipeline.add_tap(FailIfAskedTap); + + pipeline + .run(&Payload::from_map(Map::from([ + ("on", true), + ("tap_fail", true), + ]))) + .unwrap(); + let state = pipeline.state(); + assert_eq!( + state.executed(), + ["FailIfAskedFilter", "AddFilter", "FailIfAskedTap"] + ); + assert_eq!(state.errors().len(), 1); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["FailIfAskedFilter", "AddFilter"]); + + pipeline + .run(&Payload::from_map(Map::from([("fail", true)]))) + .unwrap_err(); + let state = pipeline.state(); + assert!(state.executed().is_empty()); + assert!(state.skipped().is_empty()); + assert!(state.errors().is_empty()); + let timed: Vec<&str> = state + .timings() + .iter() + .map(|(label, _)| label.as_str()) + .collect(); + assert_eq!(timed, ["FailIfAskedFilter"]); + + pipeline + .run(&Payload::from_map(Map::from([("on", false)]))) + .unwrap(); + let state = pipeline.state(); + assert_eq!(state.executed(), ["FailIfAskedFilter", "FailIfAskedTap"]); + assert_eq!(state.skipped(), ["AddFilter"]); + assert!(state.errors().is_empty()); +} + +struct FailingInnerPipeline; +impl NamedPipeline for FailingInnerPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(FailFilter); + } +} + +#[test] +fn a_failure_inside_a_nested_pipeline_is_reported_as_the_nesting_step() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(FailingInnerPipeline)); + pipeline.use_hook(hook.clone()); + assert!(pipeline.run(&Payload::new()).is_err()); + assert_eq!( + hook.events(), + [ + "before:", + "before:FailingInnerPipeline", + "error:FailingInnerPipeline", + "error:" + ] + ); +} diff --git a/packages/codeupipe-rs/tests/pricing_example.rs b/packages/codeupipe-rs/tests/pricing_example.rs new file mode 100644 index 0000000..faa3653 --- /dev/null +++ b/packages/codeupipe-rs/tests/pricing_example.rs @@ -0,0 +1,59 @@ +// The example's step files, as modules of this test. +#[path = "../examples/pricing/at_least_predicate.rs"] +mod at_least_predicate; +#[path = "../examples/pricing/audit_tap.rs"] +mod audit_tap; +#[path = "../examples/pricing/discount_filter.rs"] +mod discount_filter; +#[path = "../examples/pricing/mark_bulk_filter.rs"] +mod mark_bulk_filter; +#[path = "../examples/pricing/pricing_pipeline.rs"] +mod pricing_pipeline; + +use codeupipe::{Map, Payload, Pipeline, Value}; +use pricing_pipeline::PricingPipeline; + +#[test] +fn a_large_order_gets_the_discount_and_the_bulk_mark() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline + .run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 12.0), + ]))) + .unwrap(); + assert_eq!( + out.to_dict(), + Map::from([ + ("price", Value::from(90)), + ("qty", Value::from(12)), + ("bulk", Value::from(true)) + ]) + ); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + ); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn the_tap_reads_the_payload() { + use codeupipe::Tap; + assert!(audit_tap::AuditTap + .observe(&Payload::from_map(Map::from([("price", 90)]))) + .is_ok()); +} + +#[test] +fn a_small_order_gets_the_discount_only() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline + .run(&Payload::from_map(Map::from([ + ("price", 100.0), + ("qty", 2.0), + ]))) + .unwrap(); + assert_eq!(out.to_dict(), Map::from([("price", 90), ("qty", 2)])); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} diff --git a/packages/codeupipe-rs/tests/state_per_run.rs b/packages/codeupipe-rs/tests/state_per_run.rs new file mode 100644 index 0000000..e6a71fe --- /dev/null +++ b/packages/codeupipe-rs/tests/state_per_run.rs @@ -0,0 +1,160 @@ +// A State belongs to one run, not to the Pipeline (conformance/API.md, +// "State"): the caller can give the State, each run writes into its own, a +// step can run its own pipeline again while it runs, and state() is the +// State of the run that started last. +use std::cell::RefCell; +use std::rc::{Rc, Weak}; + +use codeupipe::{Filter, Map, Payload, Pipeline, Predicate, State, StepError, Tap, Value}; + +struct SetFilter; + +impl Filter for SetFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("a", 1); + Ok(()) + } +} + +struct IsInnerPredicate; + +impl Predicate for IsInnerPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("inner") == Some(&Value::Bool(true)) + } +} + +struct MarkFilter; + +impl Filter for MarkFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("gated", true); + Ok(()) + } +} + +struct FailTap; + +impl Tap for FailTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Err("sink down".into()) + } +} + +/// Runs the pipeline that it is in one more time, with `inner` set, and a +/// State of its own. The pipeline is given to it after it is made. +#[derive(Clone, Default)] +struct RunAgainFilter { + pipeline: Rc>>, + inner_states: Rc>>, +} + +impl Filter for RunAgainFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + if payload.get("inner") == Some(&Value::Bool(true)) { + return Ok(()); + } + let pipeline = self + .pipeline + .borrow() + .upgrade() + .expect("the pipeline is there"); + let mut state = State::new(); + pipeline.run_with_state(&Payload::from_map(Map::from([("inner", true)])), &mut state)?; + self.inner_states.borrow_mut().push(state); + Ok(()) + } +} + +/// first, again (a run inside the run), gate (only for the inner run). +fn pipeline_that_runs_again() -> (Rc, RunAgainFilter) { + let again = RunAgainFilter::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_filter(again.clone()); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + let pipeline = Rc::new(pipeline); + *again.pipeline.borrow_mut() = Rc::downgrade(&pipeline); + (pipeline, again) +} + +#[test] +fn the_caller_gives_the_state_of_a_run() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + pipeline.add_tap(FailTap); + let mut state = State::new(); + let out = pipeline + .run_with_state(&Payload::new(), &mut state) + .unwrap(); + assert_eq!(out.get("a"), Some(&Value::from(1))); + assert_eq!(state.executed(), ["SetFilter", "FailTap"]); + assert_eq!(state.skipped(), ["MarkFilter"]); + assert_eq!(state.errors()[0].1.to_string(), "sink down"); + // It is also the State of the run that started last. + assert_eq!(pipeline.state().executed(), state.executed()); + assert_eq!(pipeline.state().errors()[0].1.to_string(), "sink down"); +} + +#[test] +fn a_run_that_starts_inside_another_run_has_its_own_state() { + let (pipeline, again) = pipeline_that_runs_again(); + let mut outer = State::new(); + pipeline + .run_with_state(&Payload::new(), &mut outer) + .unwrap(); + assert_eq!(outer.executed(), ["SetFilter", "RunAgainFilter"]); + assert_eq!(outer.skipped(), ["MarkFilter"]); + let inner = again.inner_states.borrow(); + assert_eq!(inner.len(), 1); + assert_eq!( + inner[0].executed(), + ["SetFilter", "RunAgainFilter", "MarkFilter"] + ); + assert!(inner[0].skipped().is_empty()); +} + +#[test] +fn with_no_state_from_the_caller_the_pipeline_has_the_state_of_the_run_that_started_last() { + let (pipeline, _again) = pipeline_that_runs_again(); + pipeline.run(&Payload::new()).unwrap(); + // The inner run started last, inside the outer one. + assert_eq!( + pipeline.state().executed(), + ["SetFilter", "RunAgainFilter", "MarkFilter"] + ); +} + +#[test] +fn a_state_that_was_read_does_not_change_with_the_next_run() { + let mut pipeline = Pipeline::new(); + pipeline.add_valve(IsInnerPredicate, MarkFilter); + pipeline + .run(&Payload::from_map(Map::from([("inner", true)]))) + .unwrap(); + let first = pipeline.state(); + pipeline.run(&Payload::new()).unwrap(); + assert_eq!(first.executed(), ["MarkFilter"]); + assert_eq!(pipeline.state().skipped(), ["MarkFilter"]); +} + +struct FailFilter; +impl Filter for FailFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Err("boom".into()) + } +} + +#[test] +fn a_failed_run_with_the_callers_state_keeps_the_steps_before_the_failure() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(SetFilter); + pipeline.add_filter(FailFilter); + let mut state = State::new(); + assert!(pipeline + .run_with_state(&Payload::new(), &mut state) + .is_err()); + assert_eq!(state.executed(), ["SetFilter"]); + assert_eq!(pipeline.state().executed(), ["SetFilter"]); +} diff --git a/packages/codeupipe-rs/tests/step_label.rs b/packages/codeupipe-rs/tests/step_label.rs new file mode 100644 index 0000000..6ea9956 --- /dev/null +++ b/packages/codeupipe-rs/tests/step_label.rs @@ -0,0 +1,112 @@ +use std::marker::PhantomData; + +use codeupipe::{Filter, NamedPipeline, Payload, Pipeline, Predicate, StepError, Tap, Value}; + +struct AddFilter; + +impl Filter for AddFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let n = payload.get("n").and_then(Value::as_f64).unwrap_or(0.0); + payload.insert("n", n + 1.0); + Ok(()) + } +} + +struct SeeTap; + +impl Tap for SeeTap { + fn observe(&self, _payload: &Payload) -> Result<(), StepError> { + Ok(()) + } +} + +struct IsOnPredicate; + +impl Predicate for IsOnPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.get("on") == Some(&Value::Bool(true)) + } +} + +struct InnerPipeline; + +impl NamedPipeline for InnerPipeline { + fn add_steps(&self, _pipeline: &mut Pipeline) {} +} + +#[test] +fn the_label_is_the_name_of_the_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_tap(SeeTap); + assert_eq!(pipeline.labels(), ["AddFilter", "SeeTap"]); +} + +#[test] +fn the_second_use_of_a_type_gets_a_number() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + pipeline.add_filter(AddFilter); + assert_eq!( + pipeline.labels(), + ["AddFilter", "AddFilter#2", "AddFilter#3"] + ); +} + +#[test] +fn a_valve_has_the_label_of_its_filter() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.add_valve(IsOnPredicate, AddFilter); + assert_eq!(pipeline.labels(), ["AddFilter", "AddFilter#2"]); +} + +#[test] +fn a_nested_pipeline_has_the_label_of_its_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(InnerPipeline)); + // cup: allow-anonymous the label of a plain nested pipeline is what this test checks + pipeline.add_pipeline(Pipeline::new()); + assert_eq!(pipeline.labels(), ["InnerPipeline", "Pipeline"]); +} + +#[test] +fn a_type_defined_in_a_function_works() { + fn make() -> impl Filter { + struct LocalFilter; + impl Filter for LocalFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Ok(()) + } + } + LocalFilter + } + let mut pipeline = Pipeline::new(); + pipeline.add_filter(make()); + assert_eq!(pipeline.labels(), ["LocalFilter"]); +} + +struct WrapFilter(PhantomData); +impl Filter for WrapFilter { + fn call(&self, _payload: &mut Payload) -> Result<(), StepError> { + Ok(()) + } +} + +#[test] +fn a_generic_type_has_its_name_without_the_arguments() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(WrapFilter::>>( + PhantomData, + )); + assert_eq!(pipeline.labels(), ["WrapFilter"]); +} + +#[test] +fn labels_gives_a_new_list_each_time() { + let mut pipeline = Pipeline::new(); + pipeline.add_filter(AddFilter); + pipeline.labels().push("x".to_string()); + assert_eq!(pipeline.labels(), ["AddFilter"]); +} diff --git a/packages/codeupipe-rs/tests/typed_payload.rs b/packages/codeupipe-rs/tests/typed_payload.rs new file mode 100644 index 0000000..25b85c2 --- /dev/null +++ b/packages/codeupipe-rs/tests/typed_payload.rs @@ -0,0 +1,208 @@ +// A pipeline over a type of your own. Payload is the default, a map that any +// developer can read; a struct can take its place, and then every step of +// that pipeline takes the struct. +use std::cell::RefCell; +use std::rc::Rc; + +use codeupipe::{Filter, Hook, NamedPipeline, Pipeline, PipelineData, Predicate, StepError, Tap}; + +#[derive(Clone, Debug, Default, PartialEq)] +struct Order { + price: f64, + qty: u32, + bulk: bool, +} + +impl PipelineData for Order {} + +struct DiscountFilter { + rate: f64, +} + +impl Filter for DiscountFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.price = (order.price * (1.0 - self.rate) * 100.0).round() / 100.0; + Ok(()) + } +} + +struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + order.bulk = true; + Ok(()) + } +} + +struct RefuseFreeFilter; + +impl Filter for RefuseFreeFilter { + fn call(&self, order: &mut Order) -> Result<(), StepError> { + if order.price <= 0.0 { + return Err("an order must cost something".into()); + } + Ok(()) + } +} + +struct AtLeastPredicate { + minimum: u32, +} + +impl Predicate for AtLeastPredicate { + fn test(&self, order: &Order) -> bool { + order.qty >= self.minimum + } +} + +struct SeeTap { + seen: Rc>>, +} + +impl Tap for SeeTap { + fn observe(&self, order: &Order) -> Result<(), StepError> { + self.seen.borrow_mut().push(order.price); + Ok(()) + } +} + +#[derive(Clone, Default)] +struct RecordHook { + events: Rc>>, +} + +impl Hook for RecordHook { + fn before(&self, step: Option<&str>, _order: &Order) { + self.events + .borrow_mut() + .push(format!("before:{}", step.unwrap_or(""))); + } + fn on_error(&self, step: Option<&str>, _error: &StepError, order: &Order) { + self.events + .borrow_mut() + .push(format!("error:{}:{}", step.unwrap_or(""), order.price)); + } +} + +struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter { rate: 0.1 }); + pipeline.add_valve(AtLeastPredicate { minimum: 10 }, MarkBulkFilter); + } +} + +fn order(price: f64, qty: u32) -> Order { + Order { + price, + qty, + bulk: false, + } +} + +#[test] +fn every_primitive_takes_the_type() { + let seen = Rc::new(RefCell::new(Vec::new())); + let hook = RecordHook::default(); + let mut pipeline: Pipeline = Pipeline::new(); + pipeline.add_filter(DiscountFilter { rate: 0.1 }); + pipeline.add_valve(AtLeastPredicate { minimum: 10 }, MarkBulkFilter); + pipeline.add_tap(SeeTap { + seen: Rc::clone(&seen), + }); + pipeline.use_hook(hook.clone()); + + let input = order(100.0, 12); + let out = pipeline.run(&input).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 12, + bulk: true + } + ); + assert_eq!(input, order(100.0, 12)); // the caller's value is not changed + assert_eq!(*seen.borrow(), [90.0]); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "SeeTap"] + ); + assert_eq!( + *hook.events.borrow(), + ["before:", "before:DiscountFilter", "before:MarkBulkFilter"] + ); + + pipeline.run(&order(100.0, 2)).unwrap(); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} + +#[test] +fn a_named_and_nested_pipeline_takes_the_type() { + let mut pipeline = Pipeline::new(); + pipeline.add_pipeline(Pipeline::named(PricingPipeline)); + pipeline.add_filter(RefuseFreeFilter); + let out = pipeline.run(&order(50.0, 10)).unwrap(); + assert_eq!( + out, + Order { + price: 45.0, + qty: 10, + bulk: true + } + ); + assert_eq!(pipeline.labels(), ["PricingPipeline", "RefuseFreeFilter"]); +} + +#[test] +fn a_failure_is_the_same_as_with_a_payload() { + let hook = RecordHook::default(); + let mut pipeline = Pipeline::new(); + pipeline.add_filter(DiscountFilter { rate: 1.0 }); + pipeline.add_filter(RefuseFreeFilter); + pipeline.use_hook(hook.clone()); + let error = pipeline.run(&order(10.0, 1)).unwrap_err(); + assert_eq!(error.to_string(), "an order must cost something"); + assert_eq!(pipeline.state().executed(), ["DiscountFilter"]); + let events = hook.events.borrow(); + assert_eq!( + events[events.len() - 2..], + ["error:RefuseFreeFilter:0", "error::0"] + ); +} + +#[derive(Clone, Default)] +struct Traced { + steps: Vec, +} +impl PipelineData for Traced { + fn stamp(&mut self, label: &str) { + self.steps.push(label.to_string()); + } +} +struct NoopFilter; +impl Filter for NoopFilter { + fn call(&self, _traced: &mut Traced) -> Result<(), StepError> { + Ok(()) + } +} + +#[test] +fn lineage_is_kept_only_by_a_type_that_keeps_it() { + let mut traced = Pipeline::new(); + traced.add_filter(NoopFilter); + traced.observe(false, true); + assert_eq!( + traced.run(&Traced::default()).unwrap().steps, + ["NoopFilter"] + ); + + // Order does not keep lineage: with it on, the run is the same. + let mut plain = Pipeline::new(); + plain.add_filter(MarkBulkFilter); + plain.observe(true, true); + assert!(plain.run(&order(1.0, 1)).unwrap().bulk); + assert_eq!(plain.state().timings().len(), 1); +} diff --git a/packages/codeupipe-rs/tests/typed_pricing_example.rs b/packages/codeupipe-rs/tests/typed_pricing_example.rs new file mode 100644 index 0000000..35479cf --- /dev/null +++ b/packages/codeupipe-rs/tests/typed_pricing_example.rs @@ -0,0 +1,53 @@ +// The typed example's files, as modules of this test. It is the pricing +// example with an Order struct in place of the Payload, and it must give the +// same result. +#[path = "../examples/typed_pricing/at_least_predicate.rs"] +mod at_least_predicate; +#[path = "../examples/typed_pricing/audit_tap.rs"] +mod audit_tap; +#[path = "../examples/typed_pricing/discount_filter.rs"] +mod discount_filter; +#[path = "../examples/typed_pricing/mark_bulk_filter.rs"] +mod mark_bulk_filter; +#[path = "../examples/typed_pricing/order.rs"] +mod order; +#[path = "../examples/typed_pricing/pricing_pipeline.rs"] +mod pricing_pipeline; + +use codeupipe::Pipeline; +use order::Order; +use pricing_pipeline::PricingPipeline; + +#[test] +fn a_large_order_gets_the_discount_and_the_bulk_mark() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline.run(&Order::new(100.0, 12)).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 12, + bulk: true + } + ); + assert_eq!( + pipeline.state().executed(), + ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + ); + assert!(pipeline.state().errors().is_empty()); +} + +#[test] +fn a_small_order_gets_the_discount_only() { + let pipeline = Pipeline::named(PricingPipeline); + let out = pipeline.run(&Order::new(100.0, 2)).unwrap(); + assert_eq!( + out, + Order { + price: 90.0, + qty: 2, + bulk: false + } + ); + assert_eq!(pipeline.state().skipped(), ["MarkBulkFilter"]); +} diff --git a/packages/codeupipe-ts/.gitignore b/packages/codeupipe-ts/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/packages/codeupipe-ts/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/packages/codeupipe-ts/LICENSE b/packages/codeupipe-ts/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-ts/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-ts/NOTICE b/packages/codeupipe-ts/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-ts/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-ts/README.md b/packages/codeupipe-ts/README.md new file mode 100644 index 0000000..a92d274 --- /dev/null +++ b/packages/codeupipe-ts/README.md @@ -0,0 +1,216 @@ +# codeupipe for TypeScript + +codeupipe is a library for writing a program as a pipeline of small steps. +This is the TypeScript runtime. It has no dependencies. The built JavaScript +runs in a browser and in Node. + +It implements the same contract as the Python +package, and the same 30 conformance cases judge it: +[conformance/API.md](../../conformance/API.md). + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. + +The packages are not on npm yet. Make them from this repository, and +install the files that `npm pack` writes: + +```bash +git clone https://github.com/orchestrate-solutions/codeupipe +cd codeupipe && npm ci +npm pack --workspaces --pack-destination /tmp/codeupipe +``` + +```bash +# in your project +npm install /tmp/codeupipe/codeupipe-0.1.0.tgz +npm install --save-dev /tmp/codeupipe/codeupipe-lint-0.1.0.tgz /tmp/codeupipe/codeupipe-compile-0.1.0.tgz +``` + +Install the three files in one project together: the two tools need +`codeupipe` 0.1.0, and npm takes it from the file that you install. + +## The eight primitives + +| Primitive | What it does | Who writes it | +|---|---|---| +| **Payload** | Carries the data | The library | +| **Filter** | Changes the data | You | +| **Predicate** | Asks a yes-or-no question about the data | You | +| **Valve** | Joins a Predicate to a Filter: the Filter runs when the answer is yes | The library. You add one with `addValve(predicate, filter)` | +| **Tap** | Reads the data as it goes by | You | +| **Hook** | Is called before and after each step | You | +| **State** | Records which steps ran | The library | +| **Pipeline** | Runs the steps in order | You | + +## Example + +[examples/pricing/](examples/pricing/) has one class in each file. The +pipeline file imports them and adds them. + +```ts +// discount-filter.ts +import type { Filter, Payload } from "codeupipe"; + +export class DiscountFilter implements Filter { + readonly rate: number; + + constructor(rate: number) { + this.rate = rate; + } + + call(payload: Payload): Payload { + const price = payload.get("price", 0) * (1 - this.rate); + return payload.insert("price", Math.round(price * 100) / 100); + } +} +``` + +```ts +// at-least-predicate.ts +import type { Payload, Predicate } from "codeupipe"; + +export class AtLeastPredicate implements Predicate { + readonly key: string; + readonly minimum: number; + + constructor(key: string, minimum: number) { + this.key = key; + this.minimum = minimum; + } + + test(payload: Payload): boolean { + return payload.get(this.key, 0) >= this.minimum; + } +} +``` + +```ts +// pricing-pipeline.ts +import { Pipeline } from "codeupipe"; + +import { AtLeastPredicate } from "./at-least-predicate.ts"; +import { AuditTap } from "./audit-tap.ts"; +import { DiscountFilter } from "./discount-filter.ts"; +import { MarkBulkFilter } from "./mark-bulk-filter.ts"; + +export class PricingPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new DiscountFilter(0.1)); + // MarkBulkFilter runs only when the predicate is true. + this.addValve(new AtLeastPredicate("qty", 10), new MarkBulkFilter()); + // A Tap only reads. If it fails, State records the error and the run goes on. + this.addTap(new AuditTap()); + } +} +``` + +```ts +import { Payload } from "codeupipe"; + +const result = await new PricingPipeline().run(new Payload({ price: 100.0, qty: 12 })); +result.toDict(); +// { price: 90, qty: 12, bulk: true } +``` + +In this repository the example imports from `../../src/index.ts`. + +## A type of your own + +`Payload` is the default: a map that any developer can read. A class of your +own can take its place. It needs `copy()`, and the steps of that pipeline +take it: + +```ts +class Order implements PipelineData { + price: number; + constructor(price: number) { this.price = price; } + copy(): Order { return new Order(this.price); } +} + +class DiscountFilter implements Filter { + call(order: Order): Order { + order.price = Math.round(order.price * 90) / 100; + return order; + } +} + +const pipeline = new Pipeline(); +pipeline.addFilter(new DiscountFilter()); +``` + +A step for another type does not type-check. Lineage is kept only if the +class has `_stamp(label)`; otherwise `observe({ lineage: true })` does +nothing for it. The compiler reads Payload operations only, so it refuses a +step that uses `Order`. See "A type of your own" in +[conformance/API.md](../../conformance/API.md). + +## Names + +The names are the names of the contract, in camelCase. + +| Contract | TypeScript | +|---|---| +| `add_filter`, `add_valve`, `add_tap`, `add_pipeline`, `use_hook` | `addFilter`, `addValve`, `addTap`, `addPipeline`, `useHook` | +| `run`, `state`, `labels`, `observe` | `run`, `state`, `labels`, `observe({ timing, lineage })` | +| `get`, `has`, `insert`, `remove`, `merge`, `copy` | the same | +| `to_dict`, `with_trace`, `trace_id`, `lineage` | `toDict`, `withTrace`, `traceId`, `lineage` | +| Hook: `before`, `after`, `on_error` | `before`, `after`, `onError` | + +`Filter`, `Predicate` and `Tap` are interfaces: a class with the method is +one. `Hook` is a class with three methods that do nothing: extend it and +override the methods that you need. + +## Differences from Python + +- **There is one `run`, and it returns a Promise.** A Filter, a Tap or a + Hook method can be sync or async. Python has `run` (sync) and `run_async`. +- **Absent is `null`.** `get` of a key that is not there gives `null`, and a + Hook gets `null` as the step for the pipeline itself. +- **A function is not a step.** Every JavaScript function has a `call` + method, so `addFilter` refuses a function by its type. +- **Two runs of one pipeline at the same time** each write to their own + State. A caller with more than one run at a time gives each run its + State: `pipeline.run(payload, state)`. `pipeline.state` is the State of + the run that started last. +- **The linter and the compiler are separate packages**, because they + need a parser: [codeupipe-lint-ts](../codeupipe-lint-ts/) and + [codeupipe-compile-ts](../codeupipe-compile-ts/). In Python the compiler is + in the core package. + +## Labels and minified code + +State and Hooks report a step by its label, which is the name of its class. +A minifier changes class names, so a label in minified code can be a name +such as `b`. To keep the names, turn on the option of your tool that keeps +class names (for example `keepNames` in esbuild, `keep_classnames` in +terser). A class with no name has the label `anonymous`. + +`tests/bundle.test.ts` checks this with esbuild: the pricing example gives +the same result in a plain bundle, a minified bundle and a minified bundle +that keeps names, and the labels change only in the second. +`examples/pricing/browser/entry.ts` is the entry of that bundle. + +## Development + +Node 22.18 or later runs the `.ts` files as they are. + +The three TypeScript packages are one npm workspace. Run `npm ci` once, at +the root of the repository. Then, in this directory: + +```bash +npm test # the tests +npm run typecheck # the type check of the source, tests and example +npm run build # JavaScript and type files in dist/ +node examples/pricing/main.ts +python ../../conformance/run.py typescript:run +``` + +The source uses only TypeScript syntax that can be removed without a +change to the code that runs: no `enum`, no parameter properties, no +`namespace`. One primitive is in each file, and the file name is the class +name in kebab-case. + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-ts/examples/pricing/at-least-predicate.ts b/packages/codeupipe-ts/examples/pricing/at-least-predicate.ts new file mode 100644 index 0000000..3baa95d --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/at-least-predicate.ts @@ -0,0 +1,20 @@ +import type { Payload, Predicate } from "../../src/index.ts"; + +/** + * A Predicate: the question that a Valve asks. Its settings are on the + * instance, so the same class answers a different question in another + * pipeline. + */ +export class AtLeastPredicate implements Predicate { + readonly key: string; + readonly minimum: number; + + constructor(key: string, minimum: number) { + this.key = key; + this.minimum = minimum; + } + + test(payload: Payload): boolean { + return payload.get(this.key, 0) >= this.minimum; + } +} diff --git a/packages/codeupipe-ts/examples/pricing/audit-tap.ts b/packages/codeupipe-ts/examples/pricing/audit-tap.ts new file mode 100644 index 0000000..35671c3 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/audit-tap.ts @@ -0,0 +1,11 @@ +import type { Payload, Tap } from "../../src/index.ts"; + +/** + * A Tap: reads the Payload and returns nothing. If it fails, State records + * the error and the run goes on. + */ +export class AuditTap implements Tap { + observe(payload: Payload): void { + console.log("audit:", payload.toDict()); + } +} diff --git a/packages/codeupipe-ts/examples/pricing/browser/entry.ts b/packages/codeupipe-ts/examples/pricing/browser/entry.ts new file mode 100644 index 0000000..360d291 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/browser/entry.ts @@ -0,0 +1,22 @@ +// The entry of the browser check. A bundler makes one script from this file. +// The script runs the pricing pipeline and puts the result on the page and +// on globalThis, where a test can read it. +import { Payload } from "../../../src/index.ts"; +import { PricingPipeline } from "../pricing-pipeline.ts"; + +// The page of a browser. Node has none. +declare const document: { body: { textContent: string | null } } | undefined; + +const pipeline = new PricingPipeline(); +const output = (await pipeline.run(new Payload({ price: 100.0, qty: 12 }))).toDict(); +const result = { + output, + labels: pipeline.labels, + executed: pipeline.state.executed, + errors: pipeline.state.errors.map(([label, error]) => [label, String(error)]), +}; + +(globalThis as { cupResult?: unknown }).cupResult = result; +if (typeof document !== "undefined") { + document.body.textContent = JSON.stringify(result); +} diff --git a/packages/codeupipe-ts/examples/pricing/discount-filter.ts b/packages/codeupipe-ts/examples/pricing/discount-filter.ts new file mode 100644 index 0000000..19f657e --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/discount-filter.ts @@ -0,0 +1,15 @@ +import type { Filter, Payload } from "../../src/index.ts"; + +/** A Filter: takes the Payload, writes to it, returns it. */ +export class DiscountFilter implements Filter { + readonly rate: number; + + constructor(rate: number) { + this.rate = rate; + } + + call(payload: Payload): Payload { + const price = payload.get("price", 0) * (1 - this.rate); + return payload.insert("price", Math.round(price * 100) / 100); + } +} diff --git a/packages/codeupipe-ts/examples/pricing/main.ts b/packages/codeupipe-ts/examples/pricing/main.ts new file mode 100644 index 0000000..9f59a54 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/main.ts @@ -0,0 +1,15 @@ +// node examples/pricing/main.ts: run the pipeline with a Hook and show its State. +import { Payload } from "../../src/index.ts"; +import { PricingPipeline } from "./pricing-pipeline.ts"; +import { PrintStepsHook } from "./print-steps-hook.ts"; + +const pipeline = new PricingPipeline(); +pipeline.useHook(new PrintStepsHook()); + +console.log("steps:"); +const result = await pipeline.run(new Payload({ price: 100.0, qty: 12 })); + +console.log("result: ", result.toDict()); +console.log("executed:", pipeline.state.executed); +console.log("skipped: ", pipeline.state.skipped); +console.log("errors: ", pipeline.state.errors.map(([label, error]) => [label, String(error)])); diff --git a/packages/codeupipe-ts/examples/pricing/mark-bulk-filter.ts b/packages/codeupipe-ts/examples/pricing/mark-bulk-filter.ts new file mode 100644 index 0000000..8777283 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/mark-bulk-filter.ts @@ -0,0 +1,8 @@ +import type { Filter, Payload } from "../../src/index.ts"; + +/** A Filter. */ +export class MarkBulkFilter implements Filter { + call(payload: Payload): Payload { + return payload.insert("bulk", true); + } +} diff --git a/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts b/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts new file mode 100644 index 0000000..ab40e84 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/pricing-pipeline.ts @@ -0,0 +1,17 @@ +import { Pipeline } from "../../src/index.ts"; +import { AtLeastPredicate } from "./at-least-predicate.ts"; +import { AuditTap } from "./audit-tap.ts"; +import { DiscountFilter } from "./discount-filter.ts"; +import { MarkBulkFilter } from "./mark-bulk-filter.ts"; + +/** The pipeline file: the place where the steps come together. */ +export class PricingPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new DiscountFilter(0.1)); + // MarkBulkFilter runs only when the predicate is true. + this.addValve(new AtLeastPredicate("qty", 10), new MarkBulkFilter()); + // A Tap only reads. If it fails, State records the error and the run goes on. + this.addTap(new AuditTap()); + } +} diff --git a/packages/codeupipe-ts/examples/pricing/print-steps-hook.ts b/packages/codeupipe-ts/examples/pricing/print-steps-hook.ts new file mode 100644 index 0000000..bc5a933 --- /dev/null +++ b/packages/codeupipe-ts/examples/pricing/print-steps-hook.ts @@ -0,0 +1,13 @@ +import { Hook } from "../../src/index.ts"; + +/** + * A Hook: sees the lifecycle. `step` is the label of the step, which is the + * name of its class. It is null for the pipeline itself. + */ +export class PrintStepsHook extends Hook { + override before(step: string | null): void { + if (step !== null) { + console.log(" " + step); + } + } +} diff --git a/packages/codeupipe-ts/package.json b/packages/codeupipe-ts/package.json new file mode 100644 index 0000000..eed8fbf --- /dev/null +++ b/packages/codeupipe-ts/package.json @@ -0,0 +1,48 @@ +{ + "name": "codeupipe", + "version": "0.1.0", + "private": true, + "description": "A TypeScript library for writing a program as a pipeline of small steps.", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/orchestrate-solutions/codeupipe.git", + "directory": "packages/codeupipe-ts" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "LICENSE", + "NOTICE", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "scripts": { + "test": "node --test \"tests/*.test.ts\"", + "typecheck": "tsc --noEmit", + "build": "tsc -p tsconfig.build.json", + "prepack": "npm run build" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "esbuild": "^0.28.2", + "typescript": "^5.9.0" + }, + "keywords": [ + "pipeline", + "filter", + "payload", + "composition" + ] +} diff --git a/packages/codeupipe-ts/src/filter.ts b/packages/codeupipe-ts/src/filter.ts new file mode 100644 index 0000000..f621232 --- /dev/null +++ b/packages/codeupipe-ts/src/filter.ts @@ -0,0 +1,6 @@ +import type { Payload } from "./payload.ts"; + +/** Filter: changes the data. A failure throws. `call` can be async. */ +export interface Filter

{ + call(payload: P): P | Promise

; +} diff --git a/packages/codeupipe-ts/src/hook.ts b/packages/codeupipe-ts/src/hook.ts new file mode 100644 index 0000000..008b5ea --- /dev/null +++ b/packages/codeupipe-ts/src/hook.ts @@ -0,0 +1,18 @@ +import type { Payload } from "./payload.ts"; + +/** + * Hook: sees the lifecycle of a run. Extend it and override what you need. + * + * `step` is the label of the step, or null for the pipeline itself. A Hook + * reads the Payload and does not write to it. Its methods can be async. + */ +export class Hook

{ + /** Before a step, or before the pipeline starts (`step` is null). */ + before(step: string | null, payload: P): void | Promise {} + + /** After a step, or after the pipeline ends (`step` is null). */ + after(step: string | null, payload: P): void | Promise {} + + /** When a Filter fails: for the step, then for the pipeline (`step` is null). */ + onError(step: string | null, error: unknown, payload: P): void | Promise {} +} diff --git a/packages/codeupipe-ts/src/index.ts b/packages/codeupipe-ts/src/index.ts new file mode 100644 index 0000000..093c7fa --- /dev/null +++ b/packages/codeupipe-ts/src/index.ts @@ -0,0 +1,15 @@ +/** + * codeupipe: the eight primitives. + * + * Payload, Filter, Predicate, Valve, Tap, Hook, State and Pipeline, as the + * contract (conformance/API.md) defines them. A Valve is added with + * `pipeline.addValve(predicate, filter)`; its class is not a public name. + */ +export type { Filter } from "./filter.ts"; +export { Hook } from "./hook.ts"; +export { Payload } from "./payload.ts"; +export { Pipeline } from "./pipeline.ts"; +export type { PipelineData } from "./pipeline-data.ts"; +export type { Predicate } from "./predicate.ts"; +export { State } from "./state.ts"; +export type { Tap } from "./tap.ts"; diff --git a/packages/codeupipe-ts/src/payload.ts b/packages/codeupipe-ts/src/payload.ts new file mode 100644 index 0000000..3432561 --- /dev/null +++ b/packages/codeupipe-ts/src/payload.ts @@ -0,0 +1,87 @@ +/** + * Payload: the data that moves through a pipeline. + * + * One keyed container. Every write changes it in place and returns it, so a + * step can write and return in one line. `copy()` gives a separate Payload + * when you want to keep the one you have. + */ +export class Payload { // cup: allow-plain the framework itself: a primitive is made of this + #data: Map; + #traceId: string | null = null; + #lineage: string[] = []; + + /** Keys are strings; values are JSON-shaped. The map that is given is copied. */ + constructor(data?: Record | null) { + this.#data = new Map(data ? Object.entries(data) : []); + } + + /** The value for `key`, or `fallback` when the key is absent. */ + get(key: string): T | null; + get(key: string, fallback: T): T; + get(key: string, fallback: unknown = null): unknown { + return this.#data.has(key) ? this.#data.get(key) : fallback; + } + + /** Whether `key` is present (also when its value is null). */ + has(key: string): boolean { + return this.#data.has(key); + } + + /** Set `key` to `value` in this Payload, and return it. */ + insert(key: string, value: unknown): this { + this.#data.set(key, value); + return this; + } + + /** Remove `key` from this Payload (nothing happens if it is absent), and return it. */ + remove(key: string): this { + this.#data.delete(key); + return this; + } + + /** Write every key of `other` into this Payload (`other` wins), and return it. */ + merge(other: Payload): this { + for (const [key, value] of other.#data) { + this.#data.set(key, value); + } + this.#traceId = this.#traceId ?? other.#traceId; + this.#lineage.push(...other.#lineage); + return this; + } + + /** A separate Payload with the same contents, trace id and lineage. Values are shared. */ + copy(): Payload { + const copy = new Payload(); + copy.#data = new Map(this.#data); + copy.#traceId = this.#traceId; + copy.#lineage = [...this.#lineage]; + return copy; + } + + /** The contents as a new map. */ + toDict(): Record { + return Object.fromEntries(this.#data); + } + + /** Set the trace id on this Payload, and return it. */ + withTrace(traceId: string): this { + this.#traceId = traceId; + return this; + } + + /** The trace id, for following one Payload across systems. */ + get traceId(): string | null { + return this.#traceId; + } + + /** The steps this Payload has passed through, when lineage is on. */ + get lineage(): string[] { + return [...this.#lineage]; + } + + /** Record a step in the lineage. For the Pipeline. */ + _stamp(label: string): this { + this.#lineage.push(label); + return this; + } +} diff --git a/packages/codeupipe-ts/src/pipeline-data.ts b/packages/codeupipe-ts/src/pipeline-data.ts new file mode 100644 index 0000000..543e96a --- /dev/null +++ b/packages/codeupipe-ts/src/pipeline-data.ts @@ -0,0 +1,24 @@ +/** + * What a Pipeline carries from step to step. + * + * `Payload` is the default: a map that any developer can read, and that + * every runtime of codeupipe has. A class of your own can take its place, + * such as the class that your steps would use anyway: + * + * class Order implements PipelineData { + * copy(): Order { ... } + * } + * class DiscountFilter implements Filter { ... } + * const pipeline = new Pipeline(); + * + * The Pipeline asks one thing of it, `copy()`, for the copy that a run works + * on. The Filters, Predicates, Taps and Hooks of that pipeline then take the + * class, and a step for another type does not type-check. + * + * Lineage (`observe({ lineage: true })`) is kept only by a type that has + * `_stamp(label)`, as Payload has. For any other type it does nothing. + */ +export interface PipelineData

{ + copy(): P; + _stamp?(label: string): unknown; +} diff --git a/packages/codeupipe-ts/src/pipeline.ts b/packages/codeupipe-ts/src/pipeline.ts new file mode 100644 index 0000000..a9eeaa3 --- /dev/null +++ b/packages/codeupipe-ts/src/pipeline.ts @@ -0,0 +1,221 @@ +import type { Filter } from "./filter.ts"; +import type { Hook } from "./hook.ts"; +import type { Payload } from "./payload.ts"; +import type { PipelineData } from "./pipeline-data.ts"; +import type { Predicate } from "./predicate.ts"; +import { State } from "./state.ts"; +import { labelOf } from "./step-label.ts"; +import type { Tap } from "./tap.ts"; +import { Valve } from "./valve.ts"; + +type Step

> = + | { kind: "filter"; label: string; filter: Filter

} + | { kind: "valve"; label: string; valve: Valve

} + | { kind: "tap"; label: string; tap: Tap

} + | { kind: "pipeline"; label: string; pipeline: Pipeline

}; + +// The clock of browsers and of Node. Declared here so that this file needs +// the types of neither. +declare const performance: { now(): number }; + +function isThenable(value: unknown): value is PromiseLike { + return typeof (value as { then?: unknown } | null | undefined)?.then === "function"; +} + +/** Whether `value` is an object with `method`. A function is not a step: every function has `call`. */ +function has(value: unknown, method: string): boolean { + return typeof value === "object" && value !== null + && typeof (value as Record)[method] === "function"; +} + +/** Refuse a step that does not have the method of its role. */ +function require(step: unknown, method: string, what: string): void { // cup: allow-plain the framework itself: a primitive is made of this + if (!has(step, method)) { + const given = step === null || step === undefined ? String(step) + : typeof step === "function" ? "a function" : `a ${labelOf(step)}`; + throw new TypeError(`${what} has ${method}(payload). The value that was given (${given}) does not.`); + } +} + +/** + * Pipeline: runs the steps in order. + * + * Filters change the Payload, Valves gate a Filter, Taps read, Hooks see the + * lifecycle, and State records what ran. Add steps with `addFilter`, + * `addValve`, `addTap` and `addPipeline`, add Hooks with `useHook`, and run + * with `await pipeline.run(payload)`. + * + * It carries a Payload unless you give it a type of your own + * (`new Pipeline()`, see `PipelineData`). + */ +export class Pipeline

= Payload> { + #steps: Step

[] = []; + #uses: Map = new Map(); + #hooks: Hook

[] = []; + #state: State = new State(); + #timing = false; + #lineage = false; + + /** What happened in the run that started last. */ + get state(): State { + return this.#state; + } + + /** The label of each step, in order. */ + get labels(): string[] { + return this.#steps.map((step) => step.label); + } + + /** The class name of the step, with a number from its second use in this pipeline. */ + #label(step: unknown): string { + const base = labelOf(step); + const count = (this.#uses.get(base) ?? 0) + 1; + this.#uses.set(base, count); + return count === 1 ? base : `${base}#${count}`; + } + + /** Add a Filter. */ + addFilter(filter: Filter

): void { + require(filter, "call", "A Filter"); + this.#steps.push({ kind: "filter", label: this.#label(filter), filter }); + } + + /** + * Add a Valve: `filter` runs only when `predicate.test(payload)` is true. + * + * The condition comes first, then the Filter that it controls. When the + * test is false the Payload passes through unchanged and the step is + * recorded as skipped. The step has the label of its Filter. + */ + addValve(predicate: Predicate

, filter: Filter

): void { + if (!has(predicate, "test") && has(predicate, "call") && has(filter, "test")) { + throw new TypeError("The arguments are in the wrong order: addValve(predicate, filter). " + + "The condition comes first, then the Filter."); + } + require(predicate, "test", "A Predicate"); + require(filter, "call", "A Filter"); + this.#steps.push({ kind: "valve", label: this.#label(filter), valve: new Valve(predicate, filter) }); + } + + /** Add a Tap. */ + addTap(tap: Tap

): void { + require(tap, "observe", "A Tap"); + this.#steps.push({ kind: "tap", label: this.#label(tap), tap }); + } + + /** Nest a Pipeline as one step of this Pipeline. */ + addPipeline(pipeline: Pipeline

): void { + if (!(pipeline instanceof Pipeline)) { + throw new TypeError("addPipeline takes a Pipeline. The value that was given is not one."); + } + this.#steps.push({ kind: "pipeline", label: this.#label(pipeline), pipeline }); + } + + /** Add a Hook. */ + useHook(hook: Hook

): void { + this.#hooks.push(hook); + } + + /** + * Turn on timing (seconds for each step, in State) and lineage (labels on + * the Payload). A type of your own keeps lineage only if it has `_stamp`. + */ + observe(options: { timing?: boolean; lineage?: boolean } = {}): void { + this.#timing = options.timing ?? true; + this.#lineage = options.lineage ?? false; + } + + /** + * Run the steps in order on a copy of `initial`, and return the result. + * + * A step can be sync or async. A Filter failure stops the run and is + * thrown again after the Hooks have seen it. A Tap failure is recorded in + * State and the run continues. + * + * The run writes into `state` when the caller gives one, and into a new + * State when not. A State belongs to one run: a caller with more than one + * run of this Pipeline at a time gives each run its own. + */ + async run(initial: P, state: State = new State()): Promise

{ + // The State is a local of this call, so two runs at the same time do not mix. + this.#state = state; // the State of the run that started last + const hooks = this.#hooks; + const timed = this.#timing; + let payload = initial.copy(); // the caller's value is not changed + + for (const hook of hooks) { + const done = hook.before(null, payload); + if (isThenable(done)) await done; + } + + let failing: string | null = null; // the step that is running when an error escapes + let started: number | null = null; + try { + for (const step of this.#steps) { + const label = step.label; + failing = null; + started = null; + + if (step.kind === "tap") { + try { + const done = step.tap.observe(payload); + if (isThenable(done)) await done; + } catch (error) { + state.recordError(label, error); + } + state.markExecuted(label); + continue; + } + + if (timed) started = performance.now(); + failing = label; + for (const hook of hooks) { + const done = hook.before(label, payload); + if (isThenable(done)) await done; + } + + if (step.kind === "pipeline") { + payload = await step.pipeline.run(payload); + state.markExecuted(label); + } else if (step.kind === "valve" && !step.valve.predicate.test(payload)) { + state.markSkipped(label); + } else { + const filter = step.kind === "valve" ? step.valve.filter : step.filter; + const result = filter.call(payload); + payload = isThenable(result) ? await result : result; + state.markExecuted(label); + } + + for (const hook of hooks) { + const done = hook.after(label, payload); + if (isThenable(done)) await done; + } + if (timed && started !== null) state.recordTiming(label, (performance.now() - started) / 1000); + if (this.#lineage) payload._stamp?.(label); + } + } catch (error) { + if (timed && started !== null && failing !== null) { + state.recordTiming(failing, (performance.now() - started) / 1000); + } + if (failing !== null) { + // The failing step first, then the pipeline: onError fires at both + // levels, as before and after do. + for (const hook of hooks) { + const done = hook.onError(failing, error, payload); + if (isThenable(done)) await done; + } + } + for (const hook of hooks) { + const done = hook.onError(null, error, payload); + if (isThenable(done)) await done; + } + throw error; + } + + for (const hook of hooks) { + const done = hook.after(null, payload); + if (isThenable(done)) await done; + } + return payload; + } +} diff --git a/packages/codeupipe-ts/src/predicate.ts b/packages/codeupipe-ts/src/predicate.ts new file mode 100644 index 0000000..321a99c --- /dev/null +++ b/packages/codeupipe-ts/src/predicate.ts @@ -0,0 +1,11 @@ +import type { Payload } from "./payload.ts"; + +/** + * Predicate: asks a yes-or-no question about the data. + * + * A Predicate is an object. Its settings (a threshold, a flag) are on the + * instance. It reads the Payload and does not write to it. + */ +export interface Predicate

{ + test(payload: P): boolean; +} diff --git a/packages/codeupipe-ts/src/state.ts b/packages/codeupipe-ts/src/state.ts new file mode 100644 index 0000000..b96dbee --- /dev/null +++ b/packages/codeupipe-ts/src/state.ts @@ -0,0 +1,32 @@ +/** + * State: what happened in the last run. + * + * Which steps executed, which a Valve skipped, the errors, and the timings + * when timing is on. Read it after a run through `pipeline.state`. + */ +export class State { + /** Step labels, in order. */ + readonly executed: string[] = []; + /** Steps that a Valve did not run. */ + readonly skipped: string[] = []; + /** Pairs of label and error. */ + readonly errors: Array<[string, unknown]> = []; + /** Label to seconds, when timing is on. */ + readonly timings: Map = new Map(); + + markExecuted(label: string): void { + this.executed.push(label); + } + + markSkipped(label: string): void { + this.skipped.push(label); + } + + recordTiming(label: string, seconds: number): void { + this.timings.set(label, seconds); + } + + recordError(label: string, error: unknown): void { + this.errors.push([label, error]); + } +} diff --git a/packages/codeupipe-ts/src/step-label.ts b/packages/codeupipe-ts/src/step-label.ts new file mode 100644 index 0000000..79accda --- /dev/null +++ b/packages/codeupipe-ts/src/step-label.ts @@ -0,0 +1,9 @@ +/** + * The label of a step: the name of its class. State and Hooks report a step + * by this label. + */ +export function labelOf(step: unknown): string { // cup: allow-plain the framework itself: a primitive is made of this + const type = (step as { constructor?: { name?: unknown } } | null | undefined)?.constructor; + const name = type?.name; + return typeof name === "string" && name !== "" ? name : "anonymous"; +} diff --git a/packages/codeupipe-ts/src/tap.ts b/packages/codeupipe-ts/src/tap.ts new file mode 100644 index 0000000..db841fd --- /dev/null +++ b/packages/codeupipe-ts/src/tap.ts @@ -0,0 +1,9 @@ +import type { Payload } from "./payload.ts"; + +/** + * Tap: reads the data as it goes by. It does not write to the Payload. + * A Tap failure is recorded in State and never stops the run. + */ +export interface Tap

{ + observe(payload: P): void | Promise; +} diff --git a/packages/codeupipe-ts/src/valve.ts b/packages/codeupipe-ts/src/valve.ts new file mode 100644 index 0000000..ea4d335 --- /dev/null +++ b/packages/codeupipe-ts/src/valve.ts @@ -0,0 +1,18 @@ +import type { Filter } from "./filter.ts"; +import type { Predicate } from "./predicate.ts"; + +/** + * Valve: joins a Predicate to a Filter. The Filter runs only when the + * Predicate is true. + * + * `Pipeline.addValve(predicate, filter)` makes one. It is not a public name. + */ +export class Valve

{ + readonly predicate: Predicate

; + readonly filter: Filter

; + + constructor(predicate: Predicate

, filter: Filter

) { + this.predicate = predicate; + this.filter = filter; + } +} diff --git a/packages/codeupipe-ts/tests/bundle.test.ts b/packages/codeupipe-ts/tests/bundle.test.ts new file mode 100644 index 0000000..9418b33 --- /dev/null +++ b/packages/codeupipe-ts/tests/bundle.test.ts @@ -0,0 +1,53 @@ +// The built package in a bundle, as a web app has it: plain, minified, and +// minified with class names kept. The README says what a minifier does to +// labels; this file checks it. +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { pathToFileURL } from "node:url"; + +import { build } from "esbuild"; + +const ENTRY = join(import.meta.dirname, "../examples/pricing/browser/entry.ts"); +const LABELS = ["DiscountFilter", "MarkBulkFilter", "AuditTap"]; + +type Result = { output: unknown; labels: string[]; executed: string[] }; + +async function bundle(options: { minify: boolean; keepNames: boolean }): Promise<{ code: string; result: Result }> { + const built = await build({ + entryPoints: [ENTRY], bundle: true, write: false, format: "esm", platform: "browser", + target: "es2022", ...options, + }); + const code = built.outputFiles[0]?.text ?? ""; + const file = join(mkdtempSync(join(tmpdir(), "cup-bundle-")), "bundle.mjs"); + writeFileSync(file, code); + await import(pathToFileURL(file).href); + return { code, result: (globalThis as unknown as { cupResult: Result }).cupResult }; +} + +test("the bundle for a browser has no import of Node", async () => { + const { code } = await bundle({ minify: false, keepNames: false }); + assert.doesNotMatch(code, /from\s*"node:|require\(/); +}); + +test("a plain bundle gives the same result and the same labels", async () => { + const { result } = await bundle({ minify: false, keepNames: false }); + assert.deepEqual(result.output, { price: 90, qty: 12, bulk: true }); + assert.deepEqual(result.labels, LABELS); + assert.deepEqual(result.executed, LABELS); +}); + +test("a minified bundle gives the same result, but the labels are changed", async () => { + const { result } = await bundle({ minify: true, keepNames: false }); + assert.deepEqual(result.output, { price: 90, qty: 12, bulk: true }); + assert.equal(result.labels.length, 3); + assert.notDeepEqual(result.labels, LABELS); +}); + +test("a minified bundle that keeps names has the same labels", async () => { + const { result } = await bundle({ minify: true, keepNames: true }); + assert.deepEqual(result.output, { price: 90, qty: 12, bulk: true }); + assert.deepEqual(result.labels, LABELS); +}); diff --git a/packages/codeupipe-ts/tests/custom-payload.test.ts b/packages/codeupipe-ts/tests/custom-payload.test.ts new file mode 100644 index 0000000..afc27d1 --- /dev/null +++ b/packages/codeupipe-ts/tests/custom-payload.test.ts @@ -0,0 +1,177 @@ +// A pipeline over a type of your own. Payload is the default, a map that any +// developer can read; a class can take its place. The Pipeline asks one +// thing of it, copy(), for the copy that a run works on. Lineage is kept only +// by a type that has _stamp(label); for any other type it does nothing. +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { Hook, Payload, Pipeline } from "../src/index.ts"; +import type { Filter, PipelineData, Predicate, Tap } from "../src/index.ts"; + +class Order implements PipelineData { + price: number; + qty: number; + bulk: boolean; + + constructor(price: number, qty: number, bulk = false) { + this.price = price; + this.qty = qty; + this.bulk = bulk; + } + + copy(): Order { + return new Order(this.price, this.qty, this.bulk); + } +} + +class TracedOrder extends Order { + steps: string[] = []; + + override copy(): TracedOrder { + const copy = new TracedOrder(this.price, this.qty, this.bulk); + copy.steps = [...this.steps]; + return copy; + } + + _stamp(label: string): void { + this.steps.push(label); + } +} + +class DiscountFilter implements Filter { + call(order: Order): Order { + order.price = Math.round(order.price * 90) / 100; + return order; + } +} + +class MarkBulkFilter implements Filter { + call(order: Order): Order { + order.bulk = true; + return order; + } +} + +class RefuseFreeFilter implements Filter { + async call(order: Order): Promise { + if (order.price <= 0) throw new Error("an order must cost something"); + return order; + } +} + +class PassTracedFilter implements Filter { + call(order: TracedOrder): TracedOrder { + return order; + } +} + +class MarkRanFilter implements Filter { + call(payload: Payload): Payload { + return payload.insert("ran", true); + } +} + +class AtLeastPredicate implements Predicate { + test(order: Order): boolean { + return order.qty >= 10; + } +} + +class SeeTap implements Tap { + readonly seen: number[] = []; + + observe(order: Order): void { + this.seen.push(order.price); + } +} + +class RecordHook extends Hook { + readonly events: string[] = []; + + override before(step: string | null): void { + this.events.push(`before:${step ?? ""}`); + } + + override onError(step: string | null, _error: unknown, order: Order): void { + this.events.push(`error:${step ?? ""}:${order.price}`); + } +} + +class PricingPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new DiscountFilter()); + this.addValve(new AtLeastPredicate(), new MarkBulkFilter()); + } +} + +test("every primitive takes the type", async () => { + const tap = new SeeTap(); + const hook = new RecordHook(); + const pipeline = new PricingPipeline(); + pipeline.addTap(tap); + pipeline.useHook(hook); + const given = new Order(100, 12); + const out = await pipeline.run(given); + assert.deepEqual(out, new Order(90, 12, true)); + assert.deepEqual(given, new Order(100, 12)); // the caller's value is not changed + assert.deepEqual(tap.seen, [90]); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter", "SeeTap"]); + assert.deepEqual(hook.events, ["before:", "before:DiscountFilter", "before:MarkBulkFilter"]); + await pipeline.run(new Order(100, 2)); + assert.deepEqual(pipeline.state.skipped, ["MarkBulkFilter"]); +}); + +test("a nested pipeline takes the type", async () => { + const pipeline = new Pipeline(); + pipeline.addPipeline(new PricingPipeline()); + pipeline.addFilter(new RefuseFreeFilter()); + assert.deepEqual(await pipeline.run(new Order(50, 10)), new Order(45, 10, true)); +}); + +test("a failure is the same as with a Payload", async () => { + const hook = new RecordHook(); + const pipeline = new Pipeline(); + pipeline.addFilter(new DiscountFilter()); + pipeline.addFilter(new RefuseFreeFilter()); + pipeline.useHook(hook); + await assert.rejects(pipeline.run(new Order(0, 1)), /must cost something/); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter"]); + assert.deepEqual(hook.events.slice(-2), ["error:RefuseFreeFilter:0", "error::0"]); +}); + +test("lineage does nothing for a type without _stamp", async () => { + const pipeline = new PricingPipeline(); + pipeline.observe({ lineage: true }); + assert.deepEqual(await pipeline.run(new Order(100, 12)), new Order(90, 12, true)); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter"]); +}); + +test("lineage is kept by a type with _stamp", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new PassTracedFilter()); + pipeline.observe({ lineage: true }); + const given = new TracedOrder(100, 12); + const out = await pipeline.run(given); + assert.deepEqual(out.steps, ["PassTracedFilter"]); + assert.deepEqual(given.steps, []); +}); + +test("a Payload pipeline is still the default", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new MarkRanFilter()); + assert.deepEqual((await pipeline.run(new Payload())).toDict(), { ran: true }); +}); + +// Checked by the type check (npm run typecheck), never run: a pipeline +// carries one type. +export function refusedByTheTypeCheck(): void { + const payloads = new Pipeline(); + // @ts-expect-error A Filter of Order is not a Filter of Payload. + payloads.addFilter(new DiscountFilter()); + const orders = new Pipeline(); + // @ts-expect-error A pipeline of Order does not run a Payload. + void orders.run(new Payload()); + // @ts-expect-error A type of your own must have copy(). + void new Pipeline<{ price: number }>(); +} diff --git a/packages/codeupipe-ts/tests/payload.test.ts b/packages/codeupipe-ts/tests/payload.test.ts new file mode 100644 index 0000000..8170547 --- /dev/null +++ b/packages/codeupipe-ts/tests/payload.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { Payload } from "../src/index.ts"; + +test("the map that the caller gives is copied", () => { + const data = { a: 1 }; + const payload = new Payload(data); + payload.insert("b", 2); + assert.deepEqual(data, { a: 1 }); + assert.deepEqual(payload.toDict(), { a: 1, b: 2 }); +}); + +test("an empty Payload has no keys", () => { + assert.deepEqual(new Payload().toDict(), {}); +}); + +test("get gives the value, the fallback, or null", () => { + const payload = new Payload({ a: 1, n: null }); + assert.equal(payload.get("a"), 1); + assert.equal(payload.get("missing"), null); + assert.equal(payload.get("missing", 7), 7); + assert.equal(payload.get("n", 7), null); +}); + +test("has is true for a key with a null value", () => { + const payload = new Payload({ n: null }); + assert.equal(payload.has("n"), true); + assert.equal(payload.has("missing"), false); +}); + +test("insert, remove, merge and withTrace change the Payload and return it", () => { + const payload = new Payload({ a: 1 }); + assert.equal(payload.insert("b", 2), payload); + assert.equal(payload.remove("a"), payload); + assert.equal(payload.merge(new Payload({ c: 3 })), payload); + assert.equal(payload.withTrace("t-1"), payload); + assert.deepEqual(payload.toDict(), { b: 2, c: 3 }); + assert.equal(payload.traceId, "t-1"); +}); + +test("remove of a key that is not there does nothing", () => { + assert.deepEqual(new Payload({ a: 1 }).remove("zz").toDict(), { a: 1 }); +}); + +test("in a merge the other Payload wins", () => { + const payload = new Payload({ a: 1, b: 1 }).merge(new Payload({ b: 2 })); + assert.deepEqual(payload.toDict(), { a: 1, b: 2 }); +}); + +test("a merge keeps the trace id it has, and takes one when it has none", () => { + const other = new Payload({}).withTrace("other"); + assert.equal(new Payload({}).withTrace("mine").merge(other).traceId, "mine"); + assert.equal(new Payload({}).merge(other).traceId, "other"); +}); + +test("copy gives a separate Payload with the same contents", () => { + const payload = new Payload({ a: 1 }).withTrace("t-1"); + const copy = payload.copy(); + copy.insert("a", 2); + assert.equal(payload.get("a"), 1); + assert.equal(copy.traceId, "t-1"); +}); + +test("copy is shallow: the two Payloads share their values", () => { + const list = [1]; + const payload = new Payload({ list }); + assert.equal(payload.copy().get("list"), list); +}); + +test("toDict gives a new map each time", () => { + const payload = new Payload({ a: 1 }); + const out = payload.toDict(); + out.a = 2; + assert.equal(payload.get("a"), 1); +}); + +test("the trace id is null until it is set, and the lineage starts empty", () => { + const payload = new Payload({}); + assert.equal(payload.traceId, null); + assert.deepEqual(payload.lineage, []); +}); + +test("a key with the name of an Object property is only a key", () => { + const payload = new Payload({}); + assert.equal(payload.has("constructor"), false); + assert.equal(payload.get("toString"), null); + payload.insert("__proto__", 1); + assert.equal(payload.get("__proto__"), 1); + assert.deepEqual(Object.keys(payload.toDict()), ["__proto__"]); +}); diff --git a/packages/codeupipe-ts/tests/pipeline.test.ts b/packages/codeupipe-ts/tests/pipeline.test.ts new file mode 100644 index 0000000..68881dd --- /dev/null +++ b/packages/codeupipe-ts/tests/pipeline.test.ts @@ -0,0 +1,297 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { Hook, Payload, Pipeline, State } from "../src/index.ts"; +import type { Filter, Predicate, Tap } from "../src/index.ts"; + +class AddFilter implements Filter { + call(payload: Payload): Payload { + return payload.insert("n", payload.get("n", 0) + 1); + } +} + +class AddLaterFilter implements Filter { + async call(payload: Payload): Promise { + await Promise.resolve(); + return payload.insert("n", payload.get("n", 0) + 1); + } +} + +class FailFilter implements Filter { + call(): Payload { + throw new Error("boom"); + } +} + +class IsOnPredicate implements Predicate { + test(payload: Payload): boolean { + return payload.get("on") === true; + } +} + +class SeeTap implements Tap { + seen: unknown[] = []; + observe(payload: Payload): void { + this.seen.push(payload.get("n")); + } +} + +class FailTap implements Tap { + observe(): void { + throw new Error("sink is down"); + } +} + +class FailLaterTap implements Tap { + async observe(): Promise { + await Promise.resolve(); + throw new Error("sink is down later"); + } +} + +class InnerPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new AddFilter()); + this.addFilter(new AddFilter()); + } +} + +class RecordHook extends Hook { + events: string[] = []; + override before(step: string | null): void { + this.events.push("before:" + (step ?? "")); + } + override after(step: string | null): void { + this.events.push("after:" + (step ?? "")); + } + override onError(step: string | null): void { + this.events.push("error:" + (step ?? "")); + } +} + +test("an empty pipeline returns the data it was given", async () => { + const out = await new Pipeline().run(new Payload({ a: 1 })); + assert.deepEqual(out.toDict(), { a: 1 }); +}); + +test("steps run in order, and State records them", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addFilter(new AddLaterFilter()); + const out = await pipeline.run(new Payload({})); + assert.equal(out.get("n"), 2); + assert.deepEqual(pipeline.state.executed, ["AddFilter", "AddLaterFilter"]); + assert.deepEqual(pipeline.state.skipped, []); + assert.deepEqual(pipeline.state.errors, []); +}); + +test("run works on a copy: the caller's Payload is not changed", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + const given = new Payload({ n: 1 }); + const out = await pipeline.run(given); + assert.equal(given.get("n"), 1); + assert.equal(out.get("n"), 2); +}); + +test("a Valve runs its Filter when the Predicate is true", async () => { + const pipeline = new Pipeline(); + pipeline.addValve(new IsOnPredicate(), new AddFilter()); + const out = await pipeline.run(new Payload({ on: true })); + assert.equal(out.get("n"), 1); + assert.deepEqual(pipeline.state.executed, ["AddFilter"]); + assert.deepEqual(pipeline.state.skipped, []); +}); + +test("a Valve skips its Filter when the Predicate is false", async () => { + const pipeline = new Pipeline(); + pipeline.addValve(new IsOnPredicate(), new AddFilter()); + const out = await pipeline.run(new Payload({ on: false })); + assert.equal(out.has("n"), false); + assert.deepEqual(pipeline.state.executed, []); + assert.deepEqual(pipeline.state.skipped, ["AddFilter"]); +}); + +test("a Filter failure stops the run, and State keeps the steps before it", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addFilter(new FailFilter()); + pipeline.addFilter(new AddFilter()); + await assert.rejects(pipeline.run(new Payload({})), /boom/); + assert.deepEqual(pipeline.state.executed, ["AddFilter"]); +}); + +test("a Tap reads the Payload and counts as executed", async () => { + const tap = new SeeTap(); + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addTap(tap); + await pipeline.run(new Payload({})); + assert.deepEqual(tap.seen, [1]); + assert.deepEqual(pipeline.state.executed, ["AddFilter", "SeeTap"]); +}); + +test("a Tap failure is recorded and does not stop the run", async () => { + const hook = new RecordHook(); + const pipeline = new Pipeline(); + pipeline.addTap(new FailTap()); + pipeline.addTap(new FailLaterTap()); + pipeline.addFilter(new AddFilter()); + pipeline.useHook(hook); + const out = await pipeline.run(new Payload({})); + assert.equal(out.get("n"), 1); + assert.deepEqual(pipeline.state.executed, ["FailTap", "FailLaterTap", "AddFilter"]); + assert.deepEqual(pipeline.state.errors.map(([label]) => label), ["FailTap", "FailLaterTap"]); + assert.match(String(pipeline.state.errors[0]?.[1]), /sink is down/); + assert.deepEqual(hook.events, ["before:", "before:AddFilter", "after:AddFilter", "after:"]); +}); + +test("a nested pipeline is one step", async () => { + const pipeline = new Pipeline(); + pipeline.addPipeline(new InnerPipeline()); + pipeline.addFilter(new AddFilter()); + const out = await pipeline.run(new Payload({})); + assert.equal(out.get("n"), 3); + assert.deepEqual(pipeline.state.executed, ["InnerPipeline", "AddFilter"]); +}); + +test("Hooks see the pipeline and each Filter or Valve, in order", async () => { + const hook = new RecordHook(); + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addValve(new IsOnPredicate(), new AddLaterFilter()); + pipeline.addTap(new SeeTap()); + pipeline.useHook(hook); + await pipeline.run(new Payload({})); + assert.deepEqual(hook.events, [ + "before:", "before:AddFilter", "after:AddFilter", + "before:AddLaterFilter", "after:AddLaterFilter", "after:", + ]); +}); + +test("on a Filter failure a Hook gets the step, then the pipeline, and no after", async () => { + const hook = new RecordHook(); + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addFilter(new FailFilter()); + pipeline.useHook(hook); + await assert.rejects(pipeline.run(new Payload({}))); + assert.deepEqual(hook.events, [ + "before:", "before:AddFilter", "after:AddFilter", + "before:FailFilter", "error:FailFilter", "error:", + ]); +}); + +test("a Hook gets the error itself", async () => { + const seen: unknown[] = []; + class KeepErrorHook extends Hook { + override onError(_step: string | null, error: unknown): void { + seen.push(error); + } + } + const pipeline = new Pipeline(); + pipeline.addFilter(new FailFilter()); + pipeline.useHook(new KeepErrorHook()); + await assert.rejects(pipeline.run(new Payload({}))); + assert.equal(seen.length, 2); + assert.ok(seen[0] instanceof Error); + assert.equal(seen[0], seen[1]); +}); + +test("a Hook method can be async", async () => { + const events: string[] = []; + class SlowHook extends Hook { + override async before(step: string | null): Promise { + await Promise.resolve(); + events.push("before:" + (step ?? "")); + } + } + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.useHook(new SlowHook()); + await pipeline.run(new Payload({})); + assert.deepEqual(events, ["before:", "before:AddFilter"]); +}); + +test("timing and lineage are off by default", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + const out = await pipeline.run(new Payload({})); + assert.equal(pipeline.state.timings.size, 0); + assert.deepEqual(out.lineage, []); +}); + +test("observe turns timing on, and lineage when asked", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addTap(new SeeTap()); + pipeline.addValve(new IsOnPredicate(), new AddLaterFilter()); + pipeline.observe({ lineage: true }); + const out = await pipeline.run(new Payload({})); + assert.deepEqual([...pipeline.state.timings.keys()], ["AddFilter", "AddLaterFilter"]); + assert.ok((pipeline.state.timings.get("AddFilter") ?? -1) >= 0); + assert.deepEqual(out.lineage, ["AddFilter", "AddLaterFilter"]); +}); + +test("each run starts with a new State", async () => { + const pipeline = new Pipeline(); + pipeline.addValve(new IsOnPredicate(), new AddFilter()); + await pipeline.run(new Payload({ on: false })); + await pipeline.run(new Payload({ on: true })); + assert.deepEqual(pipeline.state.executed, ["AddFilter"]); + assert.deepEqual(pipeline.state.skipped, []); +}); + +test("two runs at the same time do not write into the same State", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddLaterFilter()); + pipeline.addFilter(new AddLaterFilter()); + const [a, b] = await Promise.all([pipeline.run(new Payload({})), pipeline.run(new Payload({}))]); + assert.equal(a.get("n"), 2); + assert.equal(b.get("n"), 2); + assert.deepEqual(pipeline.state.executed, ["AddLaterFilter", "AddLaterFilter#2"]); +}); + +// -- a State belongs to one run ------------------------------------------- + +class WaitFilter implements Filter { + async call(payload: Payload): Promise { + await new Promise((resolve) => setTimeout(resolve, payload.get("wait") ?? 0)); + return payload.insert("done", payload.get("name")); + } +} + +class FailForBFilter implements Filter { + call(payload: Payload): Payload { + if (payload.get("name") === "B") throw new Error("B fails"); + return payload; + } +} + +test("each of two runs at the same time writes into the State that its caller gave", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new WaitFilter()); + pipeline.addFilter(new FailForBFilter()); + const [a, b] = [new State(), new State()]; + // A starts first and ends last. B starts second, fails, and ends first. + await Promise.all([ + pipeline.run(new Payload({ name: "A", wait: 40 }), a), + pipeline.run(new Payload({ name: "B", wait: 5 }), b).catch(() => null), + ]); + assert.deepEqual(a.executed, ["WaitFilter", "FailForBFilter"]); + assert.deepEqual(b.executed, ["WaitFilter"]); + assert.equal(pipeline.state, b); // the State of the run that started last +}); + +test("with no State from the caller, a run makes a new one, and a later run does not write into a State that was given", async () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new FailForBFilter()); + const given = new State(); + await pipeline.run(new Payload({ name: "A" }), given); + assert.equal(pipeline.state, given); + await pipeline.run(new Payload({ name: "A" })); + assert.notEqual(pipeline.state, given); + assert.deepEqual(given.executed, ["FailForBFilter"]); + assert.deepEqual(pipeline.state.executed, ["FailForBFilter"]); +}); diff --git a/packages/codeupipe-ts/tests/pricing-example.test.ts b/packages/codeupipe-ts/tests/pricing-example.test.ts new file mode 100644 index 0000000..2997e77 --- /dev/null +++ b/packages/codeupipe-ts/tests/pricing-example.test.ts @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { AuditTap } from "../examples/pricing/audit-tap.ts"; +import { PricingPipeline } from "../examples/pricing/pricing-pipeline.ts"; +import { Payload } from "../src/index.ts"; + +test("a large order gets the discount and the bulk mark", async () => { + const pipeline = new PricingPipeline(); + const out = await pipeline.run(new Payload({ price: 100.0, qty: 12 })); + assert.deepEqual(out.toDict(), { price: 90, qty: 12, bulk: true }); + assert.deepEqual(pipeline.state.executed, ["DiscountFilter", "MarkBulkFilter", "AuditTap"]); + assert.deepEqual(pipeline.state.errors, []); +}); + +test("the Tap reads the Payload and prints it", (t) => { + const log = t.mock.method(console, "log", () => {}); + assert.equal(new AuditTap().observe(new Payload({ price: 90 })), undefined); + assert.deepEqual(log.mock.calls.map((call) => call.arguments), [["audit:", { price: 90 }]]); +}); + +test("a small order gets the discount only", async () => { + const pipeline = new PricingPipeline(); + const out = await pipeline.run(new Payload({ price: 100.0, qty: 2 })); + assert.deepEqual(out.toDict(), { price: 90, qty: 2 }); + assert.deepEqual(pipeline.state.skipped, ["MarkBulkFilter"]); +}); diff --git a/packages/codeupipe-ts/tests/step-label.test.ts b/packages/codeupipe-ts/tests/step-label.test.ts new file mode 100644 index 0000000..58de4d6 --- /dev/null +++ b/packages/codeupipe-ts/tests/step-label.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import * as codeupipe from "../src/index.ts"; +import { Payload, Pipeline } from "../src/index.ts"; + +class AddFilter { + call(payload: Payload): Payload { + return payload.insert("n", payload.get("n", 0) + 1); + } +} + +class SeeTap { + observe(): void {} +} + +class IsOnPredicate { + test(payload: Payload): boolean { + return payload.get("on") === true; + } +} + +class InnerPipeline extends Pipeline {} + +test("the label is the name of the class", () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addTap(new SeeTap()); + assert.deepEqual(pipeline.labels, ["AddFilter", "SeeTap"]); +}); + +test("the second use of a class gets a number", () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addFilter(new AddFilter()); + pipeline.addFilter(new AddFilter()); + assert.deepEqual(pipeline.labels, ["AddFilter", "AddFilter#2", "AddFilter#3"]); +}); + +test("a Valve has the label of its Filter", () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.addValve(new IsOnPredicate(), new AddFilter()); + assert.deepEqual(pipeline.labels, ["AddFilter", "AddFilter#2"]); +}); + +test("a nested pipeline has the label of its class", () => { + const pipeline = new Pipeline(); + pipeline.addPipeline(new InnerPipeline()); + pipeline.addPipeline(new Pipeline()); + assert.deepEqual(pipeline.labels, ["InnerPipeline", "Pipeline"]); +}); + +test("a class defined in a function works", () => { + function make() { + class LocalFilter { + call(payload: Payload): Payload { + return payload; + } + } + return new LocalFilter(); + } + const pipeline = new Pipeline(); + pipeline.addFilter(make()); + assert.deepEqual(pipeline.labels, ["LocalFilter"]); +}); + +test("a step from a class with no name has the label 'anonymous'", () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new (class { + call(payload: Payload): Payload { + return payload; + } + })()); + assert.deepEqual(pipeline.labels, ["anonymous"]); +}); + +test("labels gives a new list each time", () => { + const pipeline = new Pipeline(); + pipeline.addFilter(new AddFilter()); + pipeline.labels.push("x"); + assert.deepEqual(pipeline.labels, ["AddFilter"]); +}); + +const untyped = (value: unknown) => value as never; + +test("a step without the method of its role is refused when it is added", () => { + const pipeline = new Pipeline(); + assert.throws(() => pipeline.addFilter(untyped((p: Payload) => p)), { name: "TypeError", message: /A Filter has call\(payload\)/ }); + assert.throws(() => pipeline.addFilter(untyped(new SeeTap())), { name: "TypeError", message: /call\(payload\)/ }); + assert.throws(() => pipeline.addTap(untyped(new AddFilter())), { name: "TypeError", message: /A Tap has observe\(payload\)/ }); + assert.throws(() => pipeline.addValve(untyped(() => true), new AddFilter()), { name: "TypeError", message: /A Predicate has test\(payload\)/ }); + assert.throws(() => pipeline.addValve(new IsOnPredicate(), untyped(null)), { name: "TypeError", message: /A Filter has call\(payload\)/ }); + assert.throws(() => pipeline.addPipeline(untyped(new AddFilter())), { name: "TypeError", message: /addPipeline takes a Pipeline/ }); + assert.deepEqual(pipeline.labels, []); +}); + +test("addValve says so when the arguments are in the wrong order", () => { + const pipeline = new Pipeline(); + assert.throws(() => pipeline.addValve(untyped(new AddFilter()), untyped(new IsOnPredicate())), + { name: "TypeError", message: /wrong order/ }); +}); + +test("the package exports the primitives a user needs, and no Valve", () => { + assert.deepEqual(Object.keys(codeupipe).sort(), ["Hook", "Payload", "Pipeline", "State"]); +}); diff --git a/packages/codeupipe-ts/tsconfig.build.json b/packages/codeupipe-ts/tsconfig.build.json new file mode 100644 index 0000000..4842647 --- /dev/null +++ b/packages/codeupipe-ts/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src", + "outDir": "dist", + "declaration": true, + "types": [] + }, + "include": ["src"] +} diff --git a/packages/codeupipe-ts/tsconfig.json b/packages/codeupipe-ts/tsconfig.json new file mode 100644 index 0000000..ad97e01 --- /dev/null +++ b/packages/codeupipe-ts/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src", "tests", "examples"] +} diff --git a/packages/codeupipe/README.md b/packages/codeupipe/README.md index 6888435..9f97b61 100644 --- a/packages/codeupipe/README.md +++ b/packages/codeupipe/README.md @@ -61,7 +61,7 @@ class PricingPipeline(Pipeline): self.add_filter(DiscountFilter(0.1)) # MarkBulkFilter runs only when the predicate is true. self.add_valve(AtLeastPredicate("qty", 10), MarkBulkFilter()) - # A Tap failure is recorded and does not stop the run. + # A Tap only reads. If it fails, State records the error and the run goes on. self.add_tap(AuditTap()) ``` @@ -72,13 +72,39 @@ PricingPipeline().run(Payload({"price": 100.0, "qty": 12})).to_dict() # {'price': 90.0, 'qty': 12, 'bulk': True} ``` -A step must be a class in a module. A lambda is refused, because a step is -identified by its type, and State, Hooks and the export use that identity. +A step is a class. State and Hooks report each step by the name of its +class, for example `DiscountFilter`. A lambda is refused, because it does not +have the method of a step. + +## A type of your own + +`Payload` is the default: a map that any developer can read. Any object with +a `copy()` method can take its place, such as a dataclass: + +```python +@dataclasses.dataclass +class Order: + price: float + + def copy(self): + return dataclasses.replace(self) + + +class DiscountFilter: + def call(self, order): + order.price = round(order.price * 0.9, 2) + return order +``` + +The rules stay the same. Lineage is kept only if the object has +`_stamp(label)`; otherwise `observe(lineage=True)` does nothing for it. The +compiler reads Payload operations only: keep such a pipeline dynamic. See +"A type of your own" in [conformance/API.md](../../conformance/API.md). ## What is in the package - `codeupipe`: the eight primitives. -- `python -m codeupipe.lower`: writes a pipeline as plain Python with no +- `python -m codeupipe.compile`: writes a pipeline as plain Python with no import of codeupipe. - `codeupipe.config`: reads `cup.config.json`. @@ -88,7 +114,7 @@ in the repository README. ## Documentation - The contract: https://github.com/orchestrate-solutions/codeupipe/blob/main/conformance/API.md -- The export: https://github.com/orchestrate-solutions/codeupipe/blob/main/docs/static-export.md +- The compiler: https://github.com/orchestrate-solutions/codeupipe/blob/main/docs/compile.md - The config file: https://github.com/orchestrate-solutions/codeupipe/blob/main/docs/config.md - The repository: https://github.com/orchestrate-solutions/codeupipe diff --git a/packages/codeupipe/codeupipe/__init__.py b/packages/codeupipe/codeupipe/__init__.py index 02132a2..aca48bc 100644 --- a/packages/codeupipe/codeupipe/__init__.py +++ b/packages/codeupipe/codeupipe/__init__.py @@ -1,6 +1,6 @@ """codeupipe: the eight primitives, runnable two ways. -``codeupipe.core`` is the dynamic runtime. ``codeupipe.lower`` exports a +``codeupipe.core`` is the dynamic runtime. ``codeupipe.compile`` compiles a pipeline as plain code with no framework import. """ diff --git a/packages/codeupipe/codeupipe/lower/__init__.py b/packages/codeupipe/codeupipe/compile/__init__.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/__init__.py rename to packages/codeupipe/codeupipe/compile/__init__.py diff --git a/packages/codeupipe/codeupipe/lower/__main__.py b/packages/codeupipe/codeupipe/compile/__main__.py similarity index 89% rename from packages/codeupipe/codeupipe/lower/__main__.py rename to packages/codeupipe/codeupipe/compile/__main__.py index f4189eb..e6082fd 100644 --- a/packages/codeupipe/codeupipe/lower/__main__.py +++ b/packages/codeupipe/codeupipe/compile/__main__.py @@ -1,6 +1,6 @@ -"""``python -m codeupipe.lower [module:name --out DIR] [--lowering steps|inline] [--observe] [--overwrite] [--config FILE]`` +"""``python -m codeupipe.compile [module:name --out DIR] [--lowering steps|inline] [--observe] [--overwrite] [--config FILE]`` -Export a Pipeline to plain Python with no framework import (docs/static-export.md). +Export a Pipeline to plain Python with no framework import (docs/compile.md). With no ``module:name``, every pipeline listed under ``compile`` in ``cup.config.json`` is exported (packages/codeupipe/codeupipe/config.py). @@ -26,6 +26,7 @@ def _usage_error(message: str) -> int: return 2 +# cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result def export(target: str, out_dir: str, lowering: str = "steps", observe: bool = False, overwrite: bool = False) -> int: """Export one pipeline; the exit code for it.""" if observe and lowering == "inline": @@ -58,9 +59,10 @@ def export(target: str, out_dir: str, lowering: str = "steps", observe: bool = F return 0 +# cup: allow-plain the entry of the program: it reads the arguments, runs the pipeline and prints the result def main(argv=None) -> int: - parser = argparse.ArgumentParser(prog="python -m codeupipe.lower", - description="Export a Pipeline to plain Python (docs/static-export.md).") + parser = argparse.ArgumentParser(prog="python -m codeupipe.compile", + description="Export a Pipeline to plain Python (docs/compile.md).") parser.add_argument("target", nargs="?", help="the pipeline, as module:name (a Pipeline, a builder function, or a Pipeline type); " "omit it to export every pipeline in the config") diff --git a/packages/codeupipe/codeupipe/lower/errors.py b/packages/codeupipe/codeupipe/compile/errors.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/errors.py rename to packages/codeupipe/codeupipe/compile/errors.py diff --git a/packages/codeupipe/codeupipe/lower/filters/__init__.py b/packages/codeupipe/codeupipe/compile/filters/__init__.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/filters/__init__.py rename to packages/codeupipe/codeupipe/compile/filters/__init__.py diff --git a/packages/codeupipe/codeupipe/compile/filters/emit_inline_module_filter.py b/packages/codeupipe/codeupipe/compile/filters/emit_inline_module_filter.py new file mode 100644 index 0000000..d6d3c93 --- /dev/null +++ b/packages/codeupipe/codeupipe/compile/filters/emit_inline_module_filter.py @@ -0,0 +1,325 @@ +""" +EmitInlineModuleFilter — the whole pipeline as one flattened ``run()``. + +Direction: export (CUP -> standard Python), ``lowering="inline"``. + +The fastest static mode: steps unrolled and inlined, no framework allocations. +Measured on the spike's P1 before building this (µs per run): + + current export (call + dict copy per step) 1.021 + calls kept, one copy + in-place writes 0.849 + inlined, copies kept 0.941 + inlined + one copy + in-place writes 0.723 (hand-written: 0.728) + +Only removing both costs reaches hand-written speed, so this does both: +``run()`` copies the caller's dict once, then each step's body runs inline +and writes keys in place. + +A Payload write is in place, so a step's body is already in-place writes on +``data`` and can stand in ``run()`` as it is. A step is emitted as a function +and called instead when its shape cannot be pasted into ``run()``: an early +return, a nested function that closes over ``data``, a generator, or a step +that rebinds ``data``. +""" + +from __future__ import annotations + +import ast +import copy +import os +from typing import Dict, List, Set + +from codeupipe.core import Payload + +from ..errors import UnsupportedFilterShape + + +__all__ = ["EmitInlineModuleFilter"] + + +class EmitInlineModuleFilter: + """Emit ``//pipeline.py`` holding one flattened ``run``.""" + + def call(self, payload: Payload) -> Payload: + pipeline_ir = payload.get("pipeline_ir") + config = payload.get("config") or {"output": {"base": "src/", "step": "steps/"}} + output = config.get("output", {}) + package_dir = os.path.join(output.get("base", "src/"), output.get("step", "steps/")) + + imports: Dict[str, str] = {} + definitions: List[str] = [] + functions: List[str] = [] + body: List[str] = ["data = dict(data)"] + + for step in pipeline_ir.steps: + prefix = f"_{step.ident}__" + renames = EmitInlineModuleFilter._definition_renames(step, prefix) + helpers: Dict[str, ast.FunctionDef] = {} + for line in step.imports: + EmitInlineModuleFilter._merge_import(imports, line, step) + for source in step.definitions: + tree = ast.parse(source) + EmitInlineModuleFilter._rename_names(tree, renames) + helpers.update({n.name: n for n in tree.body if isinstance(n, ast.FunctionDef)}) + definitions.append(ast.unparse(tree)) + + call_ast = _collapse_double_dict(_renamed_copy(step.call_ast, renames)) + if step.kind == "tap": + body.extend(EmitInlineModuleFilter._tap_lines(step, call_ast, prefix, functions)) + elif step.kind == "valve": + predicate = (_collapse_double_dict(_renamed_copy(step.predicate_ast, renames)) + if step.predicate_ast else None) + if predicate is None: + raise UnsupportedFilterShape( + filter_name=step.name, method_name="predicate", line=0, col=0, + node_kind="FunctionDef", suggestion="Valve reached the emitter without a predicate.", + ) + condition = EmitInlineModuleFilter._predicate_condition(predicate, prefix, functions, helpers) + inner = EmitInlineModuleFilter._filter_lines(step, call_ast, prefix, functions) + body.append(f"if {condition}:") + body.extend(_indent(inner or ["pass"])) + else: + body.extend(EmitInlineModuleFilter._filter_lines(step, call_ast, prefix, functions)) + + body.append("return data") + sections = [ + "\n".join(sorted(imports.values())), + "\n\n".join(definitions), + "\n\n\n".join(functions), + "def run(data):\n" + "\n".join(_indent(body)), + ] + module = "\n\n\n".join(section for section in sections if section) + "\n" + + files = list(payload.get("files") or []) + files.append({"path": os.path.join(package_dir, "pipeline.py"), "content": module}) + files.append({"path": os.path.join(package_dir, "__init__.py"), + "content": "from .pipeline import run\n\n__all__ = [\"run\"]\n"}) + return payload.insert("files", files) + + @staticmethod + def _filter_lines(step, fn: ast.FunctionDef, prefix: str, functions: List[str]) -> List[str]: + """Inline a filter body, or call it as a function.""" + if EmitInlineModuleFilter._inlinable(fn, allow_final_return=True): + return EmitInlineModuleFilter._inline_filter(fn, prefix) + functions.append(_as_function(fn, f"{prefix}step")) + return [f"data = {prefix}step(data)"] + + @staticmethod + def _tap_lines(step, fn: ast.FunctionDef, prefix: str, functions: List[str]) -> List[str]: + """Plane B: a Tap failure never halts the run (Exception only).""" + if EmitInlineModuleFilter._inlinable(fn, allow_final_return=False): + inner = [ast.unparse(stmt) for stmt in EmitInlineModuleFilter._body_without_docstring(EmitInlineModuleFilter._rename_locals(fn, prefix))] + else: + functions.append(_as_function(fn, f"{prefix}tap")) + inner = [f"{prefix}tap(data)"] + return ["try:", *_indent(inner or ["pass"]), "except Exception:", " pass"] + + @staticmethod + def _predicate_condition(fn: ast.FunctionDef, prefix: str, functions: List[str], + helpers: Dict[str, ast.FunctionDef]) -> str: + body = EmitInlineModuleFilter._body_without_docstring(fn) + if len(body) == 1 and isinstance(body[0], ast.Return) and body[0].value is not None: + return ast.unparse(EmitInlineModuleFilter._inline_helper_call(body[0].value, helpers)) + functions.append(_as_function(fn, f"{prefix}predicate")) + return f"{prefix}predicate(data)" + + @staticmethod + def _inline_filter(fn: ast.FunctionDef, prefix: str) -> List[str]: + fn = EmitInlineModuleFilter._rename_locals(fn, prefix) + statements = EmitInlineModuleFilter._body_without_docstring(fn) + *head, last = statements + lines = [ast.unparse(stmt) for stmt in head] + value = last.value + if isinstance(value, ast.Name) and value.id == "data": + return lines + return lines + [f"data = {ast.unparse(value)}"] + + @staticmethod + def _inline_helper_call(expr: ast.AST, helpers: Dict[str, ast.FunctionDef]) -> ast.AST: + """``helper(dict(data))`` -> the helper's return expression over ``data``, + when the helper is ``def helper(d): return `` and only reads ``d`` + through ``d.get(...)``. Generated adapters wrap Valve predicates this + way, copying the whole dict on each check.""" + if not (isinstance(expr, ast.Call) and isinstance(expr.func, ast.Name) and len(expr.args) == 1 + and not expr.keywords and _is_dict_call(expr.args[0]) + and isinstance(expr.args[0].args[0], ast.Name) and expr.args[0].args[0].id == "data"): + return expr + helper = helpers.get(expr.func.id) + if helper is None or len(helper.args.args) != 1 or helper.args.vararg or helper.args.kwarg: + return expr + body = EmitInlineModuleFilter._body_without_docstring(helper) + if not (len(body) == 1 and isinstance(body[0], ast.Return) and body[0].value is not None): + return expr + param = helper.args.args[0].arg + returned = copy.deepcopy(body[0].value) + parents = {child: parent for parent in ast.walk(returned) for child in ast.iter_child_nodes(parent)} + for node in ast.walk(returned): + if isinstance(node, ast.Name) and node.id == param: + parent = parents.get(node) + if not (isinstance(parent, ast.Attribute) and parent.attr == "get" + and isinstance(parents.get(parent), ast.Call)): + return expr + node.id = "data" + elif isinstance(node, (ast.Lambda, ast.NamedExpr)): + return expr + return returned + + @staticmethod + def _inlinable(fn: ast.FunctionDef, allow_final_return: bool, any_return: bool = False) -> bool: + body = EmitInlineModuleFilter._body_without_docstring(fn) + if not body: + return not allow_final_return + for node in ast.walk(fn): + if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Await, ast.Global, ast.Nonlocal)): + return False + if isinstance(node, (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: + if any(isinstance(n, ast.Name) and n.id == "data" for n in ast.walk(node)): + return False + if isinstance(node, ast.Name) and node.id == "data" and not isinstance(node.ctx, ast.Load): + return False + returns = [n for n in ast.walk(fn) if isinstance(n, ast.Return)] + final = body[-1] if isinstance(body[-1], ast.Return) else None + if any_return: + pass + elif allow_final_return: + if final is None or final.value is None or returns != [final]: + return False + elif returns: + return False + return True + + @staticmethod + def _definition_renames(step, prefix: str) -> Dict[str, str]: + renames: Dict[str, str] = {} + for source in step.definitions: + for stmt in ast.parse(source).body: + for name in EmitInlineModuleFilter._top_level_bound(stmt): + renames[name] = prefix + name + return renames + + @staticmethod + def _top_level_bound(stmt: ast.stmt) -> Set[str]: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return {stmt.name} + names: Set[str] = set() + targets = stmt.targets if isinstance(stmt, ast.Assign) else [getattr(stmt, "target", None)] + for target in targets: + for node in ast.walk(target) if target is not None else (): + if isinstance(node, ast.Name): + names.add(node.id) + return names + + @staticmethod + def _rename_names(tree: ast.AST, renames: Dict[str, str]) -> None: + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id in renames: + node.id = renames[node.id] + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name in renames: + node.name = renames[node.name] + + @staticmethod + def _rename_locals(fn: ast.FunctionDef, prefix: str) -> ast.FunctionDef: + """Prefix every name the body binds, so inlined steps cannot collide.""" + fn = copy.deepcopy(fn) + local: Set[str] = set() + for node in ast.walk(fn): + if isinstance(node, ast.Name) and not isinstance(node.ctx, ast.Load): + local.add(node.id) + elif isinstance(node, ast.arg) and node is not fn.args.args[0]: + local.add(node.arg) + elif isinstance(node, ast.ExceptHandler) and node.name: + local.add(node.name) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: + local.add(node.name) + local.discard("data") + renames = {name: prefix + name for name in local} + for node in ast.walk(fn): + if isinstance(node, ast.Name) and node.id in renames: + node.id = renames[node.id] + elif isinstance(node, ast.arg) and node.arg in renames: + node.arg = renames[node.arg] + elif isinstance(node, ast.ExceptHandler) and node.name in renames: + node.name = renames[node.name] + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: + node.name = renames.get(node.name, node.name) + return fn + + @staticmethod + def _body_without_docstring(fn: ast.FunctionDef) -> List[ast.stmt]: + body = list(fn.body) + if body and isinstance(body[0], ast.Expr) and isinstance(getattr(body[0], "value", None), ast.Constant) \ + and isinstance(body[0].value.value, str): + body = body[1:] + return body + + @staticmethod + def _merge_import(imports: Dict[str, str], line: str, step) -> None: + bound = EmitInlineModuleFilter._top_level_bound_import(line) + for name in bound: + existing = imports.get(name) + if existing is not None and existing != line: + raise UnsupportedFilterShape( + filter_name=step.name, method_name="imports", line=0, col=0, node_kind="Import", + suggestion=f"`{name}` is imported differently by two steps ({existing!r} vs {line!r}); " + "use the default per-step lowering for this pipeline.", + ) + imports[name] = line + + @staticmethod + def _top_level_bound_import(line: str) -> Set[str]: + stmt = ast.parse(line).body[0] + if isinstance(stmt, ast.Import): + return {alias.asname or alias.name.split(".")[0] for alias in stmt.names} + return {alias.asname or alias.name for alias in stmt.names} + + +# -------------------------------------------------------------------------- +# steps +# -------------------------------------------------------------------------- + +def _indent(statements: List[str]) -> List[str]: + """Indent every line; an unparsed compound statement spans several.""" + return [" " + line for stmt in statements for line in stmt.splitlines()] + + +# -------------------------------------------------------------------------- +# redundant copies +# -------------------------------------------------------------------------- + +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code +class _CollapseDoubleDict(ast.NodeTransformer): + def visit_Call(self, node: ast.Call): + node = self.generic_visit(node) + if _is_dict_call(node) and _is_dict_call(node.args[0]): + return node.args[0] + return node + + +def _is_dict_call(node: ast.AST) -> bool: + return (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "dict" + and len(node.args) == 1 and not node.keywords) + + +def _collapse_double_dict(fn: ast.AST) -> ast.AST: + """``dict(dict(x))`` is ``dict(x)``.""" + return ast.fix_missing_locations(_CollapseDoubleDict().visit(fn)) + + +# -------------------------------------------------------------------------- +# inlinability: can `data` escape this body? +# -------------------------------------------------------------------------- + +# -------------------------------------------------------------------------- +# names +# -------------------------------------------------------------------------- + +def _renamed_copy(fn: ast.AST, renames: Dict[str, str]) -> ast.AST: + fn = copy.deepcopy(fn) + EmitInlineModuleFilter._rename_names(fn, renames) + return fn + + +def _as_function(fn: ast.FunctionDef, name: str) -> str: + fn = copy.deepcopy(fn) + fn.name = name + return ast.unparse(fn) diff --git a/packages/codeupipe/codeupipe/compile/filters/emit_orchestrator_filter.py b/packages/codeupipe/codeupipe/compile/filters/emit_orchestrator_filter.py new file mode 100644 index 0000000..f8a7d54 --- /dev/null +++ b/packages/codeupipe/codeupipe/compile/filters/emit_orchestrator_filter.py @@ -0,0 +1,202 @@ +""" +EmitOrchestratorFilter — the ``def run(data): ...`` glue. + +Direction: export (CUP -> standard Python). +Milestone: M1. + +Two forms, chosen by the export payload's ``observe`` flag: + +* fast (default): ``run(data)`` calls each step in order. Same output as + the dynamic run, no State, no Hooks. +* observable (``observe=True``): ``run(data, state=None, hooks=None, + timing=False, trace_id=None)`` also records State into the ``state`` dict + and calls Hooks, in exactly the order the dynamic ``Pipeline.run`` does + (conformance/API.md, "Hook" and "State"). A nested pipeline's steps are + run inline but reported as the one step that nested them. +""" + +from __future__ import annotations + +import os +from itertools import groupby +from typing import List + +from codeupipe.core import Payload + + +__all__ = ["EmitOrchestratorFilter"] + +def _default_config(payload: Payload): + return payload.get("config") or { + "pattern": payload.get("pattern", "flat"), + "output": {"base": "src/", "step": "steps/"}, + } + + +def _observed_step(name: str, body: List[str], pad: str) -> List[str]: + """A Plane A step: hooks around it, timing, the failing-step marker.""" + return [ + f"{pad}_step = {name!r}", + f"{pad}if timing:", + f"{pad} _t0 = _clock()", + f"{pad}for _hook in hooks:", + f"{pad} _hook.before({name!r}, data)", + *body, + f"{pad}for _hook in hooks:", + f"{pad} _hook.after({name!r}, data)", + f"{pad}if timing:", + f"{pad} timings[{name!r}] = _clock() - _t0", + ] + + +class EmitOrchestratorFilter: + """Emit the top-level orchestrator that calls each step in order.""" + + def call(self, payload: Payload) -> Payload: + pipeline_ir = payload.get("pipeline_ir") + config = _default_config(payload) + output = config.get("output", {}) + base = output.get("base", "src/") + step_dir = output.get("step", "steps/") + files = list(payload.get("files") or []) + + lines = EmitOrchestratorFilter._import_lines(pipeline_ir.steps) + if payload.get("observe"): + lines.extend(["", ""] + EmitOrchestratorFilter._observed_run(pipeline_ir.steps, payload.get("source_hooks") or ())) + else: + lines.extend(["", ""] + EmitOrchestratorFilter._fast_run(pipeline_ir.steps)) + lines.append("") + + files.append({ + "path": os.path.join(base, step_dir, "pipeline.py"), + "content": "\n".join(lines), + }) + files.append({ + "path": os.path.join(base, step_dir, "__init__.py"), + "content": "from .pipeline import run\n\n__all__ = [\"run\"]\n", + }) + return payload.insert("files", files) + + @staticmethod + def _import_lines(steps) -> List[str]: + # Each step lives in its own sibling module, so the orchestrator must + # import it. Relative imports keep step names (``json``, ``truncate``) + # from shadowing top-level modules on the customer's sys.path. + # Code uses each step's generated identifier (``step_03_validate``), never + # its label, reported in State and to Hooks. + lines = [] + for step in steps: + names = [step.ident] + if step.kind == "valve": + names.append(f"{step.ident}_predicate") + lines.append(f"from .{step.ident} import {', '.join(names)}") + return lines + + @staticmethod + def _plain_lines(step, pad: str, var: str = "data") -> List[str]: + """One step, no bookkeeping: the fast form, and a nested pipeline's body + (which works on its own variable, ``var``).""" + fn = step.ident + if step.kind == "valve": + return [f"{pad}if {fn}_predicate({var}):", f"{pad} {var} = {fn}({var})"] + if step.kind == "tap": + # Plane B: a Tap failure never halts the run, matching the dynamic + # runtime. Exception only; KeyboardInterrupt etc. pass. + return [f"{pad}try:", f"{pad} {fn}({var})", f"{pad}except Exception:", f"{pad} pass"] + return [f"{pad}{var} = {fn}({var})"] + + @staticmethod + def _fast_run(steps) -> List[str]: + # Steps write in place, so run() works on its own copy of the caller's dict. + lines = ["def run(data):", " data = dict(data)"] + for step in steps: + lines.extend(EmitOrchestratorFilter._plain_lines(step, " ")) + lines.append(" return data") + return lines + + @staticmethod + def _observed_run(steps, source_hooks) -> List[str]: + pad = " " + body: List[str] = [] + for group, members in groupby(steps, key=lambda s: s.group): + members = list(members) + if group is not None: + # A nested pipeline: one step in this run's State and Hooks. Its + # own skips and Tap errors belong to its run, as dynamically. + # It works on its own copy, as the nested run does: if it fails, the outer on_error hooks see ``data`` as + # it was when the nested pipeline was entered, as dynamically. + inner = [line for step in members for line in EmitOrchestratorFilter._plain_lines(step, pad, "_nested")] + body.extend(_observed_step(group, [f"{pad}_nested = dict(data)", *inner, f"{pad}data = _nested", + f"{pad}executed.append({group!r})"], pad)) + continue + for step in members: + name, fn = step.name, step.ident + if step.kind == "tap": + body.extend([ + f"{pad}_step = None", + f"{pad}try:", + f"{pad} {fn}(data)", + f"{pad}except Exception as _error:", + f"{pad} errors.append(({name!r}, _error))", + f"{pad}executed.append({name!r})", + ]) + elif step.kind == "valve": + body.extend(_observed_step(name, [ + f"{pad}if {fn}_predicate(data):", + f"{pad} data = {fn}(data)", + f"{pad} executed.append({name!r})", + f"{pad}else:", + f"{pad} skipped.append({name!r})", + ], pad)) + else: + body.extend(_observed_step(name, [ + f"{pad}data = {fn}(data)", + f"{pad}executed.append({name!r})", + ], pad)) + return [ + "from time import perf_counter as _clock", + "", + f"SOURCE_HOOKS = {tuple(source_hooks)!r}", + "", + "", + "def run(data, state=None, hooks=None, timing=False, trace_id=None):", + ' """Run the pipeline, recording State into ``state`` and calling ``hooks``.', + "", + " state: a dict to fill with executed, skipped, errors [(name, error)],", + " timings {name: seconds} (only when ``timing``) and trace_id.", + " hooks: objects with before(step, data), after(step, data) and", + " on_error(step, error, data); step is None for the pipeline itself.", + ' """', + " if hooks is None:", + " if SOURCE_HOOKS:", + " raise TypeError(", + ' "this pipeline was exported with Hooks (" + ", ".join(SOURCE_HOOKS) + "): "', + ' "pass them as run(..., hooks=[...]), or hooks=() to run without them")', + " hooks = ()", + " if state is None:", + " state = {}", + ' executed = state["executed"] = []', + ' skipped = state["skipped"] = []', + ' errors = state["errors"] = []', + ' timings = state["timings"] = {}', + ' state["trace_id"] = trace_id', + " _step = None", + " _t0 = 0.0", + " data = dict(data)", + " for _hook in hooks:", + " _hook.before(None, data)", + " try:", + *(body or [f"{pad}pass"]), + " except Exception as _error:", + " if _step is not None:", + " if timing:", + " timings[_step] = _clock() - _t0", + " for _hook in hooks:", + " _hook.on_error(_step, _error, data)", + " for _hook in hooks:", + " _hook.on_error(None, _error, data)", + " raise", + " for _hook in hooks:", + " _hook.after(None, data)", + " return data", + ] diff --git a/packages/codeupipe/codeupipe/lower/filters/emit_plain_files_filter.py b/packages/codeupipe/codeupipe/compile/filters/emit_plain_files_filter.py similarity index 65% rename from packages/codeupipe/codeupipe/lower/filters/emit_plain_files_filter.py rename to packages/codeupipe/codeupipe/compile/filters/emit_plain_files_filter.py index d5a067d..5a5ff38 100644 --- a/packages/codeupipe/codeupipe/lower/filters/emit_plain_files_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/emit_plain_files_filter.py @@ -46,10 +46,10 @@ def call(self, payload: Payload) -> Payload: # predicate of exactly that name), and the Valve predicate # would then call itself. Move the helper aside instead. wanted = f"{step.ident}_predicate" - if any(wanted in _bound_names(tree) for tree in definitions): - moved = _unused_name(f"{wanted}_impl", definitions) + if any(wanted in EmitPlainFilesFilter._bound_names(tree) for tree in definitions): + moved = EmitPlainFilesFilter._unused_name(f"{wanted}_impl", definitions) for tree in [*definitions, call_ast, predicate]: - _rename(tree, wanted, moved) + EmitPlainFilesFilter._rename(tree, wanted, moved) if isinstance(predicate, ast.FunctionDef): predicate.name = wanted @@ -69,28 +69,28 @@ def call(self, payload: Payload) -> Payload: return payload.insert("files", files) - -def _bound_names(tree: ast.AST) -> set: - names = set() - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.add(node.name) - elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): - names.add(node.id) - return names - - -def _unused_name(candidate: str, trees) -> str: - taken = set().union(*(_bound_names(t) for t in trees)) if trees else set() - name, n = candidate, 2 - while name in taken: - name, n = f"{candidate}{n}", n + 1 - return name - - -def _rename(tree: ast.AST, old: str, new: str) -> None: - for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id == old: - node.id = new - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name == old: - node.name = new + @staticmethod + def _bound_names(tree: ast.AST) -> set: + names = set() + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): + names.add(node.id) + return names + + @staticmethod + def _unused_name(candidate: str, trees) -> str: + taken = set().union(*(EmitPlainFilesFilter._bound_names(t) for t in trees)) if trees else set() + name, n = candidate, 2 + while name in taken: + name, n = f"{candidate}{n}", n + 1 + return name + + @staticmethod + def _rename(tree: ast.AST, old: str, new: str) -> None: + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id == old: + node.id = new + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name == old: + node.name = new diff --git a/packages/codeupipe/codeupipe/lower/filters/load_pipeline_filter.py b/packages/codeupipe/codeupipe/compile/filters/load_pipeline_filter.py similarity index 56% rename from packages/codeupipe/codeupipe/lower/filters/load_pipeline_filter.py rename to packages/codeupipe/codeupipe/compile/filters/load_pipeline_filter.py index d8cc10b..2aebdf9 100644 --- a/packages/codeupipe/codeupipe/lower/filters/load_pipeline_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/load_pipeline_filter.py @@ -29,30 +29,14 @@ def _refuse(name: str, node_kind: str, suggestion: str) -> UnsupportedFilterShap node_kind=node_kind, suggestion=suggestion) -def _check_pipeline(pipeline, name: str, allow_hooks: bool = False) -> None: - """Refuse a pipeline whose behaviour static code cannot reproduce. - - ``allow_hooks``: observable export (``observe=True``) calls the top-level - pipeline's Hooks. Hooks on a nested pipeline stay refused: its hooks are - objects owned by that inner run, which the flattened code does not have. - """ - if not hasattr(pipeline, "_steps"): - raise _refuse(name, type(pipeline).__name__, - "Export takes a Pipeline. This object wraps one or is not one.") - if getattr(pipeline, "_hooks", None) and not allow_hooks: - raise _refuse(name, "Hook", - "Hooks do not run in the fast static export. Export with observe=True " - "(top-level Hooks only), or remove them.") - - -def _ident(index: int, identity: str) -> str: +def _ident(index: int, label: str) -> str: """A safe, readable Python identifier for a step: ``step_03_validate``. - Generated code never uses the identity itself as a name: it is a label - (``app.filters.Validate#2``), and anything derived from user code could + Generated code never uses the label itself as a name: it is a string + (``Validate#2``), and anything derived from user code could collide with a builtin, a module global or the generated run(). """ - short = identity.rsplit("/", 1)[-1].split("#", 1)[0].rsplit(".", 1)[-1] + short = label.rsplit("/", 1)[-1].split("#", 1)[0].rsplit(".", 1)[-1] snake = re.sub(r"(? bool: return inspect.iscoroutinefunction(fn) or inspect.isasyncgenfunction(fn) -def _flatten(steps, prefix="", group=None): - """Expand nested pipelines into their steps, in order. - - A nested pipeline runs its steps in sequence on the parent's payload, so - lowering it as those steps inline is the same computation. Names get the - nesting step's name as a prefix (``inner__greet``) so they stay unique - and remain unique. Each step also carries ``group``, the - top-level step it came from, so observable export can still report the - nested pipeline as one step. - """ - for name, instance, step_type in steps: - if step_type == "pipeline": - _check_pipeline(instance, f"{prefix}{name}") - yield from _flatten(instance._steps, f"{prefix}{name}/", group or name) - else: - yield f"{prefix}{name}", instance, step_type, group - - -def _refuse_empty_nested(steps) -> None: - """A nested pipeline that flattens to no steps leaves nothing for the - observable export to hang its State entry and Hook calls on, while the - dynamic run still reports it as executed. Refuse rather than drop it.""" - for name, instance, step_type in steps: - if step_type == "pipeline" and not any(True for _ in _flatten(instance._steps)): - raise _refuse(name, "EmptyPipeline", - "A nested pipeline with no steps cannot be reported in State and Hooks " - "by observable export. Remove it, or export without observe=True.") - - class LoadPipelineFilter: """Introspect ``Pipeline._steps`` / ``._hooks`` into a skeleton IR.""" @@ -102,14 +57,14 @@ def call(self, payload: Payload) -> Payload: raise ValueError("observe=True needs the per-step layout: the inline layout writes the " "payload in place, so Hooks would see later steps' writes. Use " "lowering='steps' (the default) to observe.") - _check_pipeline(pipeline, "", allow_hooks=observe) + LoadPipelineFilter._check_pipeline(pipeline, "", allow_hooks=observe) if observe: - _refuse_empty_nested(pipeline._steps) + LoadPipelineFilter._refuse_empty_nested(pipeline._steps) steps = [] step_instances = [] step_predicates = [] - for name, instance, step_type, group in _flatten(pipeline._steps): + for name, instance, step_type, group in LoadPipelineFilter._flatten(pipeline._steps): kind = None source_instance = instance predicate = None @@ -158,3 +113,48 @@ def call(self, payload: Payload) -> Payload: .insert("source_hooks", tuple(type(h).__name__ for h in getattr(pipeline, "_hooks", ()))) ) return result + + @staticmethod + def _check_pipeline(pipeline, name: str, allow_hooks: bool = False) -> None: + """Refuse a pipeline whose behaviour static code cannot reproduce. + + ``allow_hooks``: observable export (``observe=True``) calls the top-level + pipeline's Hooks. Hooks on a nested pipeline stay refused: its hooks are + objects owned by that inner run, which the flattened code does not have. + """ + if not hasattr(pipeline, "_steps"): + raise _refuse(name, type(pipeline).__name__, + "Export takes a Pipeline. This object wraps one or is not one.") + if getattr(pipeline, "_hooks", None) and not allow_hooks: + raise _refuse(name, "Hook", + "Hooks do not run in the fast static export. Export with observe=True " + "(top-level Hooks only), or remove them.") + + @staticmethod + def _flatten(steps, prefix="", group=None): + """Expand nested pipelines into their steps, in order. + + A nested pipeline runs its steps in sequence on the parent's payload, so + lowering it as those steps inline is the same computation. Names get the + nesting step's name as a prefix (``inner__greet``) so they stay unique + and remain unique. Each step also carries ``group``, the + top-level step it came from, so observable export can still report the + nested pipeline as one step. + """ + for name, instance, step_type in steps: + if step_type == "pipeline": + LoadPipelineFilter._check_pipeline(instance, f"{prefix}{name}") + yield from LoadPipelineFilter._flatten(instance._steps, f"{prefix}{name}/", group or name) + else: + yield f"{prefix}{name}", instance, step_type, group + + @staticmethod + def _refuse_empty_nested(steps) -> None: + """A nested pipeline that flattens to no steps leaves nothing for the + observable export to hang its State entry and Hook calls on, while the + dynamic run still reports it as executed. Refuse rather than drop it.""" + for name, instance, step_type in steps: + if step_type == "pipeline" and not any(True for _ in LoadPipelineFilter._flatten(instance._steps)): + raise _refuse(name, "EmptyPipeline", + "A nested pipeline with no steps cannot be reported in State and Hooks " + "by observable export. Remove it, or export without observe=True.") diff --git a/packages/codeupipe/codeupipe/lower/filters/lower_control_flow_filter.py b/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py similarity index 82% rename from packages/codeupipe/codeupipe/lower/filters/lower_control_flow_filter.py rename to packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py index d4b8021..603612c 100644 --- a/packages/codeupipe/codeupipe/lower/filters/lower_control_flow_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/lower_control_flow_filter.py @@ -12,7 +12,7 @@ from codeupipe.core import Payload -from .rewrite_payload_ops_filter import _PayloadToDictTransformer +from .rewrite_payload_ops_filter import RewritePayloadOpsFilter, _PayloadToDictTransformer __all__ = ["LowerControlFlowFilter"] @@ -29,7 +29,8 @@ def call(self, payload: Payload) -> Payload: predicate_ast = step.predicate_ast if predicate_ast is not None: name = getattr(predicate_ast, "name", "predicate") - transformer = _PayloadToDictTransformer(step.name, name, step.ident) + transformer = _PayloadToDictTransformer( + step.name, name, step.ident, RewritePayloadOpsFilter.payload_parameter(predicate_ast, after_self=False)) predicate_ast = ast.fix_missing_locations( transformer.visit(ast.parse(ast.unparse(predicate_ast)).body[0]) ) diff --git a/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py b/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py new file mode 100644 index 0000000..49d3b0c --- /dev/null +++ b/packages/codeupipe/codeupipe/compile/filters/parse_filter_source_filter.py @@ -0,0 +1,419 @@ +""" +ParseFilterSourceFilter — extract each Filter's ``call`` method as an AST. + +Direction: export (CUP -> standard Python). +Milestone: M0. +""" + +from __future__ import annotations + +import ast +import builtins +import inspect +import sys +import textwrap +import types +from dataclasses import replace +from typing import Iterable, Optional, Tuple + +from codeupipe.core import Payload + +from ..errors import UnsupportedFilterShape + + +__all__ = ["ParseFilterSourceFilter"] + + +_BUILTIN_NAMES = frozenset(dir(builtins)) + + +class ParseFilterSourceFilter: + """Turn each step's live instance into an ``ast.FunctionDef``.""" + + def call(self, payload: Payload) -> Payload: + pipeline_ir = payload.get("pipeline_ir") + step_instances = payload.get("step_instances") or [] + step_predicates = payload.get("step_predicates") or [None] * len(step_instances) + rewritten_steps = [] + + for step, instance, predicate in zip(pipeline_ir.steps, step_instances, step_predicates): + method_name = "observe" if step.kind == "tap" else "call" + method = getattr(instance, method_name, None) + if method is None: + raise UnsupportedFilterShape( + filter_name=step.name, + method_name=method_name, + line=getattr(step.call_ast, "lineno", 0), + col=getattr(step.call_ast, "col_offset", 0), + node_kind="Attribute", + suggestion="Define the step in a real .py file with a concrete call/observe method.", + ) + try: + source = textwrap.dedent(inspect.getsource(method)) + except (OSError, TypeError): + raise UnsupportedFilterShape( + filter_name=step.name, + method_name=method_name, + line=getattr(step.call_ast, "lineno", 0), + col=getattr(step.call_ast, "col_offset", 0), + node_kind="FunctionDef", + suggestion="Define the step in a real .py file so inspect.getsource can recover it.", + ) + function_ast = _erase_annotations(ast.parse(source).body[0]) + function_ast, config = ParseFilterSourceFilter._bind_instance_config(function_ast, instance, step) + imports, definitions = ParseFilterSourceFilter._detect_dependencies(function_ast, method, step.name) + definitions = config + definitions + predicate_ast = None + if step.kind == "valve": + predicate_ast, p_config = ParseFilterSourceFilter._parse_predicate(step, predicate) + p_imports, p_definitions = ParseFilterSourceFilter._detect_dependencies(predicate_ast, predicate.test, step.name) + imports = tuple(sorted(set(imports) | set(p_imports))) + p_definitions = p_config + p_definitions + definitions = definitions + tuple(d for d in p_definitions if d not in definitions) + rewritten_steps.append( + replace( + step, + call_ast=function_ast, + docstring=ast.get_docstring(function_ast), + imports=imports, + definitions=definitions, + predicate_ast=predicate_ast, + ) + ) + + return payload.insert("pipeline_ir", replace(pipeline_ir, steps=rewritten_steps)) + + @staticmethod + def _bind_instance_config(fn: ast.FunctionDef, instance, step, prefix: str = "_self_") -> Tuple[ast.FunctionDef, tuple]: + """Compile a step's constructor config into it as named constants. + + Export lowers a *live* Pipeline, so each ``self.`` the method reads + already has its value. A value with a literal form is baked in as + ``_self_ = `` — per step, so two instances of one class + keep their own config. Anything that cannot be compiled in is refused + here rather than left as a reference to a ``self`` that will not exist. + """ + if not fn.args.args: + return fn, () + self_name = fn.args.args[0].arg + constants: dict = {} + + def refuse(attr: str, why: str, node: ast.AST) -> UnsupportedFilterShape: + return UnsupportedFilterShape( + filter_name=step.name, + method_name=fn.name, + line=getattr(node, "lineno", 0), + col=getattr(node, "col_offset", 0), + node_kind="Attribute", + suggestion=f"`{self_name}.{attr}` {why}", + ) + + class _Bind(ast.NodeTransformer): + def visit_Attribute(self, node: ast.Attribute): + if not (isinstance(node.value, ast.Name) and node.value.id == self_name): + return self.generic_visit(node) + attr = node.attr + if not isinstance(node.ctx, ast.Load): + raise refuse(attr, "is written inside the step; per-instance state has no static form.", node) + if attr not in constants: + if not hasattr(instance, attr): + raise refuse(attr, "is not set on the instance being exported.", node) + value = getattr(instance, attr) + if callable(value): + raise refuse(attr, "is a method or callable; inline it, or make it a module-level function.", node) + literal = ParseFilterSourceFilter._literal_source(value) + if literal is None: + raise refuse(attr, f"holds a {type(value).__name__}, which has no literal form to compile in.", node) + constants[attr] = (f"{prefix}{attr}", literal) + return ast.copy_location(ast.Name(id=constants[attr][0], ctx=ast.Load()), node) + + def visit_Name(self, node: ast.Name): + if node.id == self_name: + raise refuse("", f"`{self_name}` is used directly; only reads of literal attributes can be compiled in.", node) + return node + + body = [_Bind().visit(stmt) for stmt in fn.body] + fn.body = [ast.fix_missing_locations(stmt) for stmt in body] + definitions = tuple(f"{name} = {literal}" for name, literal in constants.values()) + return fn, definitions + + @staticmethod + def _literal_source(value) -> Optional[str]: + """``repr(value)`` if it evaluates back to an equal value of the same type.""" + source = repr(value) + try: + back = ast.literal_eval(source) + except (ValueError, SyntaxError, TypeError, MemoryError, RecursionError): + return None + if type(back) is not type(value) or back != value: + return None + return source + + @staticmethod + def _parse_predicate(step, predicate) -> Tuple[ast.FunctionDef, tuple]: + """Recover a Valve predicate object's ``test`` as a ``def`` of its Payload. + + Predicates are objects (conformance/API.md, "Valve"): the ``test`` + method is lowered like a filter's ``call``, the instance's literal + settings compiled in as constants (``_when_``, so they cannot + collide with the filter's ``_self_`` in the same module). The + orchestrator always emits ``if _predicate(data):``, so a predicate + that cannot be recovered stops the export here. + """ + def refuse(suggestion: str, kind: str = "FunctionDef") -> UnsupportedFilterShape: + return UnsupportedFilterShape( + filter_name=step.name, method_name="predicate", line=0, col=0, node_kind=kind, suggestion=suggestion, + ) + + method = getattr(predicate, "test", None) + if method is None: + raise refuse("A Valve predicate is an object with test(self, payload) -> bool.") + try: + fn = _erase_annotations(ast.parse(textwrap.dedent(inspect.getsource(method))).body[0]) + except (OSError, TypeError, SyntaxError, IndexError): + raise refuse("Define the predicate class in a real .py file so inspect.getsource can recover it.") + if not isinstance(fn, ast.FunctionDef) or len(fn.args.args) != 2: + raise refuse("A Valve predicate's test must be a plain `def test(self, payload)`.") + fn, config = ParseFilterSourceFilter._bind_instance_config(fn, predicate, step, prefix="_when_") + # self is gone: its reads are now constants. The parameter keeps the name + # that the author gave it; RewritePayloadOps follows that name. + fn.args.args = fn.args.args[1:] + return fn, config + + @staticmethod + def _detect_dependencies(fn: ast.FunctionDef, obj, step_name: str) -> Tuple[tuple, tuple]: + """Return ``(imports, definitions)`` the step body needs to run standalone. + + Names are resolved against the defining module's *source* first, because + that is what the author wrote: a top-level import is re-emitted as that + import, and a top-level constant, helper, or class is copied verbatim + (with whatever it needs in turn). Only names the source does not bind + fall back to runtime introspection of the value. + + Resolving by value alone is what dropped same-module constants and + turned ``_WORDS = re.compile(...)`` into ``from re import _WORDS``: a + compiled pattern reports ``__module__ == "re"``. A name that neither + route can place raises instead of being dropped. + """ + module = inspect.getmodule(obj) + if module is None: + return (), () + module_ns = getattr(module, "__dict__", {}) + if not module_ns: + return (), () + bindings = ParseFilterSourceFilter._module_bindings(module) + + imports: set = set() + definitions: dict = {} + pending = list(ParseFilterSourceFilter._referenced_free_names(fn, ParseFilterSourceFilter._local_names(fn))) + seen: set = set() + while pending: + name = pending.pop() + if name in seen or name in _BUILTIN_NAMES: + continue + seen.add(name) + + stmt = bindings.get(name) + if isinstance(stmt, (ast.Import, ast.ImportFrom)): + import_line = ParseFilterSourceFilter._import_line_for_alias(stmt, name) + if import_line is not None: + imports.add(import_line) + continue + elif stmt is not None: + definitions[stmt.lineno] = ast.unparse(stmt) + pending.extend(ParseFilterSourceFilter._free_names(stmt)) + continue + + if name not in module_ns: + continue + import_line = ParseFilterSourceFilter._import_statement_for(name, module_ns[name], module.__name__) + if import_line is None: + raise UnsupportedFilterShape( + filter_name=step_name, + method_name=fn.name, + line=getattr(fn, "lineno", 0), + col=getattr(fn, "col_offset", 0), + node_kind="Name", + suggestion=( + f"`{name}` has no top-level definition or import in {module.__name__}'s source " + "and cannot be imported by name; define it at module level so export can copy it." + ), + ) + imports.add(import_line) + + return tuple(sorted(imports)), tuple(definitions[line] for line in sorted(definitions)) + + @staticmethod + def _module_bindings(module) -> dict: + """Map each name to the last top-level statement in the module that binds it.""" + try: + tree = ast.parse(inspect.getsource(module)) + except (OSError, TypeError, SyntaxError): + return {} + bindings: dict = {} + for stmt in tree.body: + if isinstance(stmt, ast.Import): + for alias in stmt.names: + bindings[alias.asname or alias.name.split(".")[0]] = stmt + elif isinstance(stmt, ast.ImportFrom): + for alias in stmt.names: + bindings[alias.asname or alias.name] = stmt + elif isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + bindings[stmt.name] = stmt + elif isinstance(stmt, ast.Assign): + for target in stmt.targets: + names: set = set() + ParseFilterSourceFilter._collect_target_names(target, names) + for name in names: + bindings[name] = stmt + elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None: + if isinstance(stmt.target, ast.Name): + bindings[stmt.target.id] = stmt + return bindings + + @staticmethod + def _import_line_for_alias(stmt, name: str) -> Optional[str]: + """Re-emit only the alias that binds ``name``; relative imports return None.""" + if isinstance(stmt, ast.ImportFrom): + if stmt.level or stmt.module == "__future__": + return None + for alias in stmt.names: + if (alias.asname or alias.name) == name: + suffix = f" as {alias.asname}" if alias.asname else "" + return f"from {stmt.module} import {alias.name}{suffix}" + return None + for alias in stmt.names: + if (alias.asname or alias.name.split(".")[0]) == name: + return f"import {alias.name} as {alias.asname}" if alias.asname else f"import {alias.name}" + return None + + @staticmethod + def _free_names(stmt: ast.AST) -> set: + """Names a copied top-level statement loads but does not itself bind.""" + loaded: set = set() + bound: set = set() + for node in ast.walk(stmt): + if isinstance(node, ast.Name): + (loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id) + elif isinstance(node, ast.arg): + bound.add(node.arg) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + bound.add(node.name) + return loaded - bound + + @staticmethod + def _local_names(fn: ast.FunctionDef) -> set: + names = {"self", "cls"} + args = fn.args + for arg in list(args.args) + list(args.posonlyargs) + list(args.kwonlyargs): + names.add(arg.arg) + if args.vararg is not None: + names.add(args.vararg.arg) + if args.kwarg is not None: + names.add(args.kwarg.arg) + for node in ast.walk(fn): + if isinstance(node, ast.Assign): + for target in node.targets: + ParseFilterSourceFilter._collect_target_names(target, names) + elif isinstance(node, ast.AugAssign) or isinstance(node, ast.AnnAssign): + ParseFilterSourceFilter._collect_target_names(node.target, names) + elif isinstance(node, (ast.For, ast.comprehension)): + ParseFilterSourceFilter._collect_target_names(node.target, names) + elif isinstance(node, ast.With): + for item in node.items: + if item.optional_vars is not None: + ParseFilterSourceFilter._collect_target_names(item.optional_vars, names) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: + names.add(node.name) + return names + + @staticmethod + def _collect_target_names(target: ast.AST, names: set) -> None: + if isinstance(target, ast.Name): + names.add(target.id) + elif isinstance(target, (ast.Tuple, ast.List)): + for elt in target.elts: + ParseFilterSourceFilter._collect_target_names(elt, names) + elif isinstance(target, ast.Starred): + ParseFilterSourceFilter._collect_target_names(target.value, names) + + @staticmethod + def _referenced_free_names(fn: ast.FunctionDef, local_names: set) -> Iterable[str]: + seen: set = set() + for node in ast.walk(fn): + root = None + if isinstance(node, ast.Name): + root = node.id + elif isinstance(node, ast.Attribute): + root = ParseFilterSourceFilter._leftmost_name(node) + if root and root not in local_names and root not in seen: + seen.add(root) + yield root + + @staticmethod + def _leftmost_name(node: ast.Attribute) -> Optional[str]: + current = node.value + while isinstance(current, ast.Attribute): + current = current.value + if isinstance(current, ast.Name): + return current.id + return None + + @staticmethod + def _import_statement_for(local_name: str, value, current_module: str) -> Optional[str]: + if isinstance(value, types.ModuleType): + real_name = value.__name__ + if real_name == local_name: + return f"import {real_name}" + return f"import {real_name} as {local_name}" + + origin = getattr(value, "__module__", None) + if not origin or origin == current_module or origin == "builtins": + return None + real_name = getattr(value, "__name__", local_name) + # ``__module__`` names where a value's *type* lives, not where the value + # can be imported from (a compiled pattern reports "re"). Only emit the + # import if it would actually hand back this object. + if getattr(sys.modules.get(origin), real_name, None) is not value: + return None + if real_name == local_name: + return f"from {origin} import {local_name}" + return f"from {origin} import {real_name} as {local_name}" + + +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code +class _EraseAnnotations(ast.NodeTransformer): + """Drop every type hint in a function: on its parameters, its return, and + its locals (``x: T = v`` becomes ``x = v``; a bare ``x: T`` goes).""" + + def _function(self, node): + node.returns = None + arguments = node.args + for argument in arguments.posonlyargs + arguments.args + arguments.kwonlyargs: + argument.annotation = None + for argument in (arguments.vararg, arguments.kwarg): + if argument is not None: + argument.annotation = None + return self.generic_visit(node) + + visit_FunctionDef = visit_AsyncFunctionDef = _function + + def generic_visit(self, node): + node = super().generic_visit(node) + # A block whose only statement was a bare hint must not be left empty. + if isinstance(getattr(node, "body", None), list) and not node.body: + node.body = [ast.Pass()] + return node + + def visit_AnnAssign(self, node): + if node.value is None: + return None + return ast.copy_location(ast.Assign(targets=[node.target], value=self.visit(node.value)), node) + + +def _erase_annotations(fn): + """A step's type hints say what it reads and writes; they are for the + author and the tools, as in TypeScript. Exported code carries none, so it + never refers to the hint's names (``Payload``, a TypedDict) and so never + imports the framework.""" + return ast.fix_missing_locations(_EraseAnnotations().visit(fn)) diff --git a/packages/codeupipe/codeupipe/lower/filters/rewrite_payload_ops_filter.py b/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py similarity index 84% rename from packages/codeupipe/codeupipe/lower/filters/rewrite_payload_ops_filter.py rename to packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py index c3b807c..c686714 100644 --- a/packages/codeupipe/codeupipe/lower/filters/rewrite_payload_ops_filter.py +++ b/packages/codeupipe/codeupipe/compile/filters/rewrite_payload_ops_filter.py @@ -36,21 +36,6 @@ def _name(identifier: str, ctx=None) -> ast.Name: _FRAMEWORK_MODULES = ("codeupipe", "codeupipe") -def _drop_unused_framework_imports(imports: tuple, *trees: ast.AST) -> tuple: - """Lowering removes ``Payload`` uses; drop their imports - so no framework import survives into the emitted package.""" - used = {n.id for tree in trees if tree is not None for n in ast.walk(tree) if isinstance(n, ast.Name)} - kept = [] - for line in imports: - stmt = ast.parse(line).body[0] - module = getattr(stmt, "module", None) or (stmt.names[0].name if isinstance(stmt, ast.Import) else "") - bound = {a.asname or a.name.split(".")[0] for a in stmt.names} - if module.split(".")[0] in _FRAMEWORK_MODULES and not (bound & used): - continue - kept.append(line) - return tuple(kept) - - # Payload members this rewriter lowers. Anything else read off the payload # (``trace_id``, ``lineage``, ``with_trace``, ``_data``) has no plain-dict # counterpart and would otherwise pass through as ``data.`` on a dict, @@ -67,6 +52,7 @@ def _drop_unused_framework_imports(imports: tuple, *trees: ast.AST) -> tuple: "of a return, or as the value assigned to a name. Move it out of the larger expression.") +# cup: allow-plain a visitor of a syntax tree: the tool that a Filter uses to read or change source code class _PayloadToDictTransformer(ast.NodeTransformer): """Lower a step's Payload operations to operations on a plain dict. @@ -76,12 +62,13 @@ class _PayloadToDictTransformer(ast.NodeTransformer): or a ``copy()``) becomes a name bound to a dict. """ - def __init__(self, filter_name: str, method_name: str, fn_name: str = ""): - self.filter_name = filter_name # the step's identity: for messages + def __init__(self, filter_name: str, method_name: str, fn_name: str = "", parameter: str = "payload"): + self.filter_name = filter_name # the step's label: for messages self.method_name = method_name self.fn_name = fn_name or filter_name # the generated function's name # source name bound to a Payload -> the dict variable it becomes - self.payloads = {"payload": "data"} + self.parameter = parameter # the name the step gives its Payload + self.payloads = {parameter: "data"} self._temps = 0 # -- which expressions are Payloads --------------------------------- @@ -178,7 +165,7 @@ def visit_Assign(self, node: ast.Assign): source = node.targets[0].id statements: list = [] value = self._lower_payload(node.value, statements) - variable = self.payloads.setdefault(source, "data" if source == "payload" else source) + variable = self.payloads.setdefault(source, "data" if source == self.parameter else source) if not (isinstance(value, ast.Name) and value.id == variable): statements.append(ast.Assign(targets=[_name(variable, ast.Store())], value=value)) return self._located(statements or [ast.Pass()], node) @@ -198,10 +185,10 @@ def visit_Attribute(self, node: ast.Attribute): and node.attr not in _LOWERED_MEMBERS): if node.attr in _OBSERVABILITY_MEMBERS: raise self._error(node, "Attribute", - f"payload.{node.attr} has no static form: exported steps work on a " + f"{node.value.id}.{node.attr} has no static form: exported steps work on a " "plain dict, which carries no trace ID or lineage.") raise self._error(node, "Attribute", - f"payload.{node.attr} is not an exportable Payload operation " + f"{node.value.id}.{node.attr} is not an exportable Payload operation " f"({', '.join(sorted(_LOWERED_MEMBERS))}).") return self.generic_visit(node) @@ -272,16 +259,43 @@ def call(self, payload: Payload) -> Payload: rewritten_steps = [] for step in pipeline_ir.steps: - call_transformer = _PayloadToDictTransformer(step.name, step.call_ast.name, step.ident) + call_transformer = _PayloadToDictTransformer(step.name, step.call_ast.name, step.ident, + RewritePayloadOpsFilter.payload_parameter(step.call_ast)) call_ast = ast.fix_missing_locations(call_transformer.visit(ast.parse(ast.unparse(step.call_ast)).body[0])) predicate_ast = step.predicate_ast if predicate_ast is not None: predicate_name = getattr(predicate_ast, "name", "predicate") - predicate_transformer = _PayloadToDictTransformer(step.name, predicate_name, step.ident) + predicate_transformer = _PayloadToDictTransformer( + step.name, predicate_name, step.ident, + RewritePayloadOpsFilter.payload_parameter(predicate_ast, after_self=False)) predicate_ast = ast.fix_missing_locations( predicate_transformer.visit(ast.parse(ast.unparse(predicate_ast)).body[0]) ) - imports = _drop_unused_framework_imports(tuple(step.imports or ()), call_ast, predicate_ast) + imports = RewritePayloadOpsFilter._drop_unused_framework_imports(tuple(step.imports or ()), call_ast, predicate_ast) rewritten_steps.append(replace(step, call_ast=call_ast, predicate_ast=predicate_ast, imports=imports)) return payload.insert("pipeline_ir", replace(pipeline_ir, steps=rewritten_steps)) + + @staticmethod + def payload_parameter(fn: ast.FunctionDef, after_self: bool = True) -> str: + """The name that a step gives its Payload: the parameter after ``self``, + or the first one for a Valve's predicate, whose ``self`` is already gone + (its reads are constants). A step that calls it ``order`` compiles as + one that calls it ``payload``.""" + at = 1 if after_self else 0 + return fn.args.args[at].arg if len(fn.args.args) > at else "payload" + + @staticmethod + def _drop_unused_framework_imports(imports: tuple, *trees: ast.AST) -> tuple: + """Lowering removes ``Payload`` uses; drop their imports + so no framework import survives into the emitted package.""" + used = {n.id for tree in trees if tree is not None for n in ast.walk(tree) if isinstance(n, ast.Name)} + kept = [] + for line in imports: + stmt = ast.parse(line).body[0] + module = getattr(stmt, "module", None) or (stmt.names[0].name if isinstance(stmt, ast.Import) else "") + bound = {a.asname or a.name.split(".")[0] for a in stmt.names} + if module.split(".")[0] in _FRAMEWORK_MODULES and not (bound & used): + continue + kept.append(line) + return tuple(kept) diff --git a/packages/codeupipe/codeupipe/lower/ir.py b/packages/codeupipe/codeupipe/compile/ir.py similarity index 96% rename from packages/codeupipe/codeupipe/lower/ir.py rename to packages/codeupipe/codeupipe/compile/ir.py index c90426a..88d7a64 100644 --- a/packages/codeupipe/codeupipe/lower/ir.py +++ b/packages/codeupipe/codeupipe/compile/ir.py @@ -92,7 +92,7 @@ class SymbolFlowShapeIR: @dataclass(frozen=True) -class StepIR: +class StepIR: # cup: allow-plain a record of data: it checks its own fields when it is made """A single pipeline step in canonical form. Attributes: @@ -113,8 +113,8 @@ class StepIR: emitted file runs without that module. ident: The Python identifier the generated code uses for this step's module and function (``step_03_validate``). Generated, never taken - from ``name``: ``name`` is the step's identity (a label such as - ``app.filters.Validate#2``), which is not an identifier. + from ``name``: ``name`` is the step's label (such as + ``Validate#2``), which is not an identifier. group: For a step flattened out of a nested Pipeline, the name of the top-level step that nested it (``inner`` for ``inner__greet``). Observable export uses it to report the nested pipeline as one diff --git a/packages/codeupipe/codeupipe/lower/pipelines/__init__.py b/packages/codeupipe/codeupipe/compile/pipelines/__init__.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/pipelines/__init__.py rename to packages/codeupipe/codeupipe/compile/pipelines/__init__.py diff --git a/packages/codeupipe/codeupipe/lower/pipelines/export_pipeline.py b/packages/codeupipe/codeupipe/compile/pipelines/export_pipeline.py similarity index 70% rename from packages/codeupipe/codeupipe/lower/pipelines/export_pipeline.py rename to packages/codeupipe/codeupipe/compile/pipelines/export_pipeline.py index 17042d1..96a6984 100644 --- a/packages/codeupipe/codeupipe/lower/pipelines/export_pipeline.py +++ b/packages/codeupipe/codeupipe/compile/pipelines/export_pipeline.py @@ -19,10 +19,10 @@ from .lowering_mode_predicate import LoweringModePredicate -__all__ = ["build_export_pipeline_v2"] +__all__ = ["ExportPipeline", "build_export_pipeline_v2"] -def build_export_pipeline_v2(*, write: bool = True) -> Pipeline: +class ExportPipeline(Pipeline): """Assemble the v2 export pipeline. Args: @@ -42,20 +42,26 @@ def build_export_pipeline_v2(*, write: bool = True) -> Pipeline: State or Hooks. True: run() also records State and calls Hooks as the dynamic run does; "steps" - layout only. docs/static-export.md + layout only. docs/compile.md Output payload keys: ``pipeline_ir`` (PipelineIR) — the canonical IR. ``files`` (list[dict]) — the emitted files. """ - p = Pipeline() - p.add_filter(LoadPipelineFilter()) - p.add_filter(ParseFilterSourceFilter()) - p.add_filter(RewritePayloadOpsFilter()) - p.add_filter(LowerControlFlowFilter()) - p.add_valve(LoweringModePredicate(inline=False), EmitPlainFilesFilter()) - p.add_valve(LoweringModePredicate(inline=False), EmitOrchestratorFilter()) - p.add_valve(LoweringModePredicate(inline=True), EmitInlineModuleFilter()) - if write: - p.add_tap(WriteFilesTap()) - return p + + def __init__(self, *, write: bool = True): + super().__init__() + self.add_filter(LoadPipelineFilter()) + self.add_filter(ParseFilterSourceFilter()) + self.add_filter(RewritePayloadOpsFilter()) + self.add_filter(LowerControlFlowFilter()) + self.add_valve(LoweringModePredicate(inline=False), EmitPlainFilesFilter()) + self.add_valve(LoweringModePredicate(inline=False), EmitOrchestratorFilter()) + self.add_valve(LoweringModePredicate(inline=True), EmitInlineModuleFilter()) + if write: + self.add_tap(WriteFilesTap()) + + +def build_export_pipeline_v2(*, write: bool = True) -> Pipeline: + """The export pipeline. Kept for callers that used this name: see ``ExportPipeline``.""" + return ExportPipeline(write=write) diff --git a/packages/codeupipe/codeupipe/lower/pipelines/lowering_mode_predicate.py b/packages/codeupipe/codeupipe/compile/pipelines/lowering_mode_predicate.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/pipelines/lowering_mode_predicate.py rename to packages/codeupipe/codeupipe/compile/pipelines/lowering_mode_predicate.py diff --git a/packages/codeupipe/codeupipe/lower/resolve_target.py b/packages/codeupipe/codeupipe/compile/resolve_target.py similarity index 93% rename from packages/codeupipe/codeupipe/lower/resolve_target.py rename to packages/codeupipe/codeupipe/compile/resolve_target.py index d5c3845..fbe450d 100644 --- a/packages/codeupipe/codeupipe/lower/resolve_target.py +++ b/packages/codeupipe/codeupipe/compile/resolve_target.py @@ -14,6 +14,7 @@ class TargetError(Exception): """The ``module:name`` given does not lead to a Pipeline.""" +# cup: allow-plain used by the entry of the program: it finds the pipeline that an argument names def resolve_target(spec: str) -> Pipeline: """``name`` may be a Pipeline, or anything callable with no arguments that returns one: a builder function or a named Pipeline type.""" diff --git a/packages/codeupipe/codeupipe/lower/taps/__init__.py b/packages/codeupipe/codeupipe/compile/taps/__init__.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/taps/__init__.py rename to packages/codeupipe/codeupipe/compile/taps/__init__.py diff --git a/packages/codeupipe/codeupipe/lower/taps/write_files_tap.py b/packages/codeupipe/codeupipe/compile/taps/write_files_tap.py similarity index 100% rename from packages/codeupipe/codeupipe/lower/taps/write_files_tap.py rename to packages/codeupipe/codeupipe/compile/taps/write_files_tap.py diff --git a/packages/codeupipe/codeupipe/config.py b/packages/codeupipe/codeupipe/config.py index 35ce5f1..847cd8b 100644 --- a/packages/codeupipe/codeupipe/config.py +++ b/packages/codeupipe/codeupipe/config.py @@ -27,6 +27,7 @@ class ConfigError(Exception): """The config file cannot be used as it is.""" +# cup: allow-plain it reads the config file, before any pipeline exists def _unknown(where: str, given, allowed) -> None: extra = sorted(set(given) - set(allowed)) if extra: @@ -34,11 +35,13 @@ def _unknown(where: str, given, allowed) -> None: where, ", ".join(repr(key) for key in extra), ", ".join(allowed))) +# cup: allow-plain it reads the config file, before any pipeline exists def _strings(where: str, value) -> None: if not (isinstance(value, list) and all(isinstance(item, str) for item in value)): raise ConfigError("%s must be a list of strings" % where) +# cup: allow-plain it reads the config file, before any pipeline exists def load_config(path: Optional[str] = None) -> Optional[dict]: """The config as a dict, checked. With no ``path``: ``cup.config.json`` in the working directory, or None when there is no such file.""" diff --git a/packages/codeupipe/codeupipe/core/README.md b/packages/codeupipe/codeupipe/core/README.md index c2099bb..07a40be 100644 --- a/packages/codeupipe/codeupipe/core/README.md +++ b/packages/codeupipe/codeupipe/core/README.md @@ -2,7 +2,7 @@ This package holds the eight primitives: `Payload`, `Filter`, `Predicate`, `Valve`, `Tap`, `Hook`, `State`, `Pipeline`. The contract is -[conformance/API.md](../../../../conformance/API.md). The export (`codeupipe.lower`) and the linter +[conformance/API.md](../../../../conformance/API.md). The compiler (`codeupipe.compile`) and the linter (`codeupipe-lint`) are built on this package. No dependencies: standard library only. @@ -16,14 +16,13 @@ library only. `add_valve`, `add_tap`, `add_pipeline`, `use_hook`, `run`, `run_async`, `state`. `run()` is synchronous and never starts an event loop; `run_async()` is the opt-in - entry point for async steps. Steps have no - names: each is identified by its filter (`step_identity.py`), `#n` from - the second use; State and Hooks report that identity. Nested pipelines are - named `Pipeline` subclasses. See conformance/API.md, "Step identity". -- `step_identity.py` — derives a step's identity (`module.QualName`) and the - `file:line` it was added at; refuses anonymous steps. -- `anonymous_step.py` — `AnonymousStep` (a `TypeError`): lambdas, closures, - classes defined in a function, `__main__` filters, unnamed nested pipelines. + entry point for async steps. Steps are not named by + hand: each has a label, the name of its class (`step_label.py`), `#n` from + the second use; State and Hooks report that label. A step without the + method of its role is refused with a `TypeError` when it is added. See + conformance/API.md, "Step labels". +- `step_label.py` — `label_of(step)`: the class name of a step; for a Valve, + of its Filter. - `async_step_in_sync_run.py` — `AsyncStepInSyncRun`: raised when the sync `Pipeline.run()` meets async work (it never starts an event loop). - `predicate.py` — `Predicate`: a yes-or-no question about the payload, an @@ -35,15 +34,16 @@ library only. - `tap.py` — `Tap` / `AsyncTap`: read-only observation `Protocol`s (`observe(payload) -> None`); must not modify the payload. - `state.py` — `State`: pipeline execution metadata — executed/skipped - filters, timing, errors, arbitrary metadata. Available as `pipeline.state` - after a run. + filters, timing, errors, arbitrary metadata. A State belongs to one run. + `pipeline.state` is the State of the run that started last. A caller with + more than one run at a time gives each run its own: `run(payload, state)`. - `hook.py` — `Hook`: lifecycle ABC with `before`/`after`/`on_error` callbacks around filter and pipeline execution. - `async_hook.py` — `AsyncHook`: a Hook whose methods are coroutines. ## How it fits in -`core` imports nothing else from codeupipe. `codeupipe.lower` and +`core` imports nothing else from codeupipe. `codeupipe.compile` and `codeupipe-lint` import from it, never the reverse. The data flow the framework is built around is `Payload → Pipeline.run(payload) → Filter.call(payload) → Payload`. diff --git a/packages/codeupipe/codeupipe/core/__init__.py b/packages/codeupipe/codeupipe/core/__init__.py index 5d17fd2..aea534f 100644 --- a/packages/codeupipe/codeupipe/core/__init__.py +++ b/packages/codeupipe/codeupipe/core/__init__.py @@ -9,7 +9,6 @@ from .payload import Payload from .filter import AsyncFilter, Filter from .async_step_in_sync_run import AsyncStepInSyncRun -from .anonymous_step import AnonymousStep from .pipeline import Pipeline from .predicate import Predicate from .tap import AsyncTap, Tap @@ -25,5 +24,5 @@ "Tap", "AsyncTap", "Hook", "AsyncHook", "State", - "AnonymousStep", "AsyncStepInSyncRun", + "AsyncStepInSyncRun", ] diff --git a/packages/codeupipe/codeupipe/core/anonymous_step.py b/packages/codeupipe/codeupipe/core/anonymous_step.py deleted file mode 100644 index e78d547..0000000 --- a/packages/codeupipe/codeupipe/core/anonymous_step.py +++ /dev/null @@ -1,13 +0,0 @@ -"""AnonymousStep: a step whose identity cannot be derived from the code. - -Step identity is a pointer to the filter (conformance/API.md, "Step -identity"). Lambdas, closures, classes defined inside a function, filters -defined in the entry script and unnamed nested pipelines have no stable -pointer, and are not independently testable, so the runtime refuses them. -""" - -__all__ = ["AnonymousStep"] - - -class AnonymousStep(TypeError): - """Raised when a step has no stable identity to report.""" diff --git a/packages/codeupipe/codeupipe/core/payload.py b/packages/codeupipe/codeupipe/core/payload.py index 5a9be90..82c14f5 100644 --- a/packages/codeupipe/codeupipe/core/payload.py +++ b/packages/codeupipe/codeupipe/core/payload.py @@ -13,6 +13,7 @@ T = TypeVar('T') +# cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them class Payload(Generic[T]): """Keyed data. Keys are strings; values are JSON-shaped. diff --git a/packages/codeupipe/codeupipe/core/pipeline.py b/packages/codeupipe/codeupipe/core/pipeline.py index 3d9b63a..091de6a 100644 --- a/packages/codeupipe/codeupipe/core/pipeline.py +++ b/packages/codeupipe/codeupipe/core/pipeline.py @@ -13,7 +13,7 @@ from .tap import AsyncTap, Tap from .hook import Hook from .state import State -from .step_identity import add_site, identify +from .step_label import label_of from .async_step_in_sync_run import AsyncStepInSyncRun from .valve import Valve @@ -24,6 +24,7 @@ TOutput = TypeVar('TOutput') +# cup: allow-plain a primitive itself: the framework makes the primitives, so it is not written with them class Pipeline(Generic[TInput, TOutput]): """ Orchestrator — runs filters in sequence with hooks, taps, and state tracking. @@ -37,11 +38,9 @@ class Pipeline(Generic[TInput, TOutput]): def __init__(self): self._preflight_key: Any = None self._preflight_cache: Any = None - self._steps: List[Tuple[str, Any, str]] = [] # (identity, step, type) - # Step identity (conformance/API.md): derived from the code, #n from - # the second use of the same filter; the add call's file:line as detail. - self._occurrences: Dict[str, int] = {} - self._step_sites: Dict[str, str] = {} + self._steps: List[Tuple[str, Any, str]] = [] # (label, step, type) + # A step's label is its class name, with #n from the second use. + self._uses: Dict[str, int] = {} self._hooks: List[Hook] = [] self._state: State = State() self._observe_timing: bool = False @@ -49,54 +48,61 @@ def __init__(self): @property def state(self) -> State: - """Access pipeline execution state after run().""" - return self._state + """The State of the run that started last. - @property - def identities(self) -> List[str]: - """Each step's identity, in order.""" - return [identity for identity, _step, _type in self._steps] + A State belongs to one run. A caller with more than one run of this + Pipeline at a time gives each run its own: ``run(payload, state)``. + """ + return self._state @property - def step_sites(self) -> Dict[str, str]: - """identity -> ``file:line`` of the ``add_*`` call that added it.""" - return dict(self._step_sites) - - def _identity(self, step: Any, kind: str, base: Optional[str] = None) -> str: - """Derive the step's identity; number it from its second use.""" - base = base or identify(step, kind) - count = self._occurrences.get(base, 0) + 1 - self._occurrences[base] = count - identity = base if count == 1 else f"{base}#{count}" - self._step_sites[identity] = add_site(3) # the caller of add_* + def labels(self) -> List[str]: + """Each step's label, in order.""" + return [label for label, _step, _type in self._steps] + + def _label(self, step: Any) -> str: + """The step's class name, numbered from its second use in this pipeline.""" + base = label_of(step) + count = self._uses.get(base, 0) + 1 + self._uses[base] = count self._preflight_key = None # shape changed - return identity + return base if count == 1 else f"{base}#{count}" + + @staticmethod + def _require(step: Any, method: str, what: str) -> None: + """Refuse a step that does not have the method of its kind.""" + if not callable(getattr(step, method, None)): + raise TypeError(f"{what} has {method}(payload). {step!r} does not.") def add_filter(self, filter: Union[Filter[TInput, TOutput], AsyncFilter[TInput, TOutput]]) -> None: """Add a Filter. A filter whose ``call`` is a coroutine needs ``run_async()``.""" if isinstance(filter, Valve): raise TypeError("A Valve is added with add_valve(predicate, filter), not with add_filter.") + self._require(filter, "call", "A Filter") step_type = "async_filter" if self._is_async_method(filter, "call") else "filter" - self._steps.append((self._identity(filter, "filter"), filter, step_type)) + self._steps.append((self._label(filter), filter, step_type)) def add_valve(self, predicate: Any, filter: Union[Filter[TInput, TOutput], AsyncFilter[TInput, TOutput]]) -> None: """Add a Valve: ``filter`` runs only when ``predicate.test(payload)`` is true. The condition comes first, then the Filter that it controls. When the test is false the Payload passes through unchanged and the step is - recorded as skipped. The step is identified by its Filter. + recorded as skipped. The step has the label of its Filter. """ if not callable(getattr(predicate, "test", None)) and callable(getattr(predicate, "call", None)) \ and callable(getattr(filter, "test", None)): raise TypeError("The arguments are in the wrong order: add_valve(predicate, filter). " "The condition comes first, then the Filter.") + self._require(predicate, "test", "A Predicate") + self._require(filter, "call", "A Filter") valve = Valve(filter, predicate) - self._steps.append((self._identity(valve, "filter"), valve, "async_filter")) + self._steps.append((self._label(valve), valve, "valve")) def add_tap(self, tap: Union[Tap, AsyncTap]) -> None: """Add a Tap. A tap whose ``observe`` is a coroutine needs ``run_async()``.""" + self._require(tap, "observe", "A Tap") step_type = "async_tap" if self._is_async_method(tap, "observe") else "tap" - self._steps.append((self._identity(tap, "tap"), tap, step_type)) + self._steps.append((self._label(tap), tap, step_type)) def use_hook(self, hook: Hook) -> None: """Attach a lifecycle hook. A hook with coroutine methods needs ``run_async()``.""" @@ -104,8 +110,10 @@ def use_hook(self, hook: Hook) -> None: self._hooks.append(hook) def add_pipeline(self, pipeline: 'Pipeline') -> None: - """Nest a named Pipeline type as a single step inside this Pipeline.""" - self._steps.append((self._identity(pipeline, "pipeline"), pipeline, "pipeline")) + """Nest a Pipeline as a single step inside this Pipeline.""" + if not isinstance(pipeline, Pipeline): + raise TypeError(f"add_pipeline takes a Pipeline. {pipeline!r} is not one.") + self._steps.append((self._label(pipeline), pipeline, "pipeline")) def observe(self, *, timing: bool = True, lineage: bool = False) -> None: """Enable observation features (timing, lineage tracking).""" @@ -129,37 +137,50 @@ def _can_run_fast(self) -> bool: """Whether the common case can skip hooks and instrumentation.""" return not self._hooks and not self._observe_timing and not self._observe_lineage - async def _run_fast(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: + async def _run_fast(self, initial_payload: Payload[TInput], state: State) -> Payload[TOutput]: """Hot path for simple pipelines: no hooks, no timing, no lineage.""" - self._state = State() payload = initial_payload.copy() # the caller's Payload is not changed for name, step, step_type in self._steps: if step_type == "tap": - self._observe(name, step, payload) - self._state.mark_executed(name) + self._observe(name, step, payload, state) + state.mark_executed(name) continue if step_type == "async_tap": - await self._observe_async(name, step, payload) - self._state.mark_executed(name) + await self._observe_async(name, step, payload, state) + state.mark_executed(name) continue if step_type == "pipeline": payload = await step.run_async(payload) - self._state.mark_executed(name) + state.mark_executed(name) continue - if step_type == "async_filter": + if step_type == "valve": + # The Pipeline asks the Predicate itself. A flag on the Valve + # would be shared by each run that is in progress. + if not step._predicate(payload): + state.mark_skipped(name) + continue + result = step._inner.call(payload) + payload = await result if inspect.isawaitable(result) else result + elif step_type == "async_filter": payload = await step.call(payload) # type: ignore[attr-defined] else: payload = step.call(payload) # type: ignore[attr-defined] - if hasattr(step, "_last_skipped") and step._last_skipped: - self._state.mark_skipped(name) - else: - self._state.mark_executed(name) + state.mark_executed(name) return payload # type: ignore[return-value] + @staticmethod + def _stamp(payload: Any, label: str) -> None: + """Lineage: record that a step ran. Payload keeps the label. A type of + your own keeps it only if it has ``_stamp(label)``; for any other type, + lineage does nothing. The return value of ``_stamp`` is not used.""" + stamp = getattr(payload, "_stamp", None) + if stamp is not None: + stamp(label) + @staticmethod async def _invoke(fn, *args): """Call fn(*args), awaiting the result only if it is a coroutine.""" @@ -168,12 +189,19 @@ async def _invoke(fn, *args): result = await result return result - async def run_async(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: - """Execute with async steps allowed (opt-in). ``run()`` is the default.""" + async def run_async(self, initial_payload: Payload[TInput], state: Optional[State] = None) -> Payload[TOutput]: + """Execute with async steps allowed (opt-in). ``run()`` is the default. + + The run writes into ``state`` when the caller gives one, and into a + new State when not. It is a local of this call, so two runs that are + in progress at the same time never write into the same State. + """ + if state is None: + state = State() + self._state = state # the State of the run that started last if self._can_run_fast(): - return await self._run_fast(initial_payload) + return await self._run_fast(initial_payload, state) - self._state = State() payload = initial_payload.copy() # the caller's Payload is not changed _step_name: Optional[str] = None _step_t0: Optional[float] = None @@ -189,13 +217,13 @@ async def run_async(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: _failing = None if step_type == "tap": - self._observe(name, step, payload) - self._state.mark_executed(name) + self._observe(name, step, payload, state) + state.mark_executed(name) continue if step_type == "async_tap": - await self._observe_async(name, step, payload) - self._state.mark_executed(name) + await self._observe_async(name, step, payload, state) + state.mark_executed(name) continue _step_t0 = time.monotonic() @@ -206,30 +234,32 @@ async def run_async(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: if step_type == "pipeline": payload = await step.run_async(payload) - self._state.mark_executed(name) + state.mark_executed(name) + elif step_type == "valve": + if step._predicate(payload): + result = step._inner.call(payload) + payload = await result if inspect.isawaitable(result) else result + state.mark_executed(name) + else: + state.mark_skipped(name) else: - # filter or valve if step_type == "async_filter": payload = await step.call(payload) # type: ignore[attr-defined] else: payload = step.call(payload) # type: ignore[attr-defined] - - if hasattr(step, '_last_skipped') and step._last_skipped: - self._state.mark_skipped(name) - else: - self._state.mark_executed(name) + state.mark_executed(name) for hook in self._hooks: await self._invoke(hook.after, name, payload) if self._observe_timing: - self._state.record_timing(name, time.monotonic() - _step_t0) + state.record_timing(name, time.monotonic() - _step_t0) if self._observe_lineage: - payload = payload._stamp(name) + self._stamp(payload, name) except Exception as e: if _step_t0 is not None and _step_name is not None and self._observe_timing: - self._state.record_timing(_step_name, time.monotonic() - _step_t0) + state.record_timing(_step_name, time.monotonic() - _step_t0) if _failing is not None: # The failing step first, then the pipeline: on_error mirrors # before/after, which fire at both levels. @@ -244,7 +274,7 @@ async def run_async(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: return payload # type: ignore - def _observe(self, name: str, tap: Any, payload: Payload) -> None: + def _observe(self, name: str, tap: Any, payload: Payload, state: State) -> None: """Run a sync Tap on Plane B: its failure never halts Plane A. The ``Exception`` is recorded in ``state.errors`` under the Tap's @@ -255,24 +285,27 @@ def _observe(self, name: str, tap: Any, payload: Payload) -> None: try: tap.observe(payload) except Exception as error: - self._state.record_error(name, error) + state.record_error(name, error) - async def _observe_async(self, name: str, tap: Any, payload: Payload) -> None: + async def _observe_async(self, name: str, tap: Any, payload: Payload, state: State) -> None: """``_observe`` for a Tap whose ``observe`` may be a coroutine.""" try: result = tap.observe(payload) if inspect.isawaitable(result): await result except Exception as error: - self._state.record_error(name, error) + state.record_error(name, error) # ------------------------------------------------------------------ # Sync execution: the default. Never starts an event loop. # ------------------------------------------------------------------ - def run(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: + def run(self, initial_payload: Payload[TInput], state: Optional[State] = None) -> Payload[TOutput]: """Execute synchronously (the default). + The run writes into ``state`` when the caller gives one, and into a + new State when not (see ``run_async``). + Never starts an event loop: if the pipeline contains async work (an async filter, tap or hook) this raises ``AsyncStepInSyncRun``; use ``run_async()`` for those pipelines. Entering the async path costs @@ -284,7 +317,7 @@ def run(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: f"cannot run async step '{offender}' inside run(). Use " f"run_async() from async code, or replace it with a sync step." ) - return self._run_sync(initial_payload) + return self._run_sync(initial_payload, state) def _preflight(self) -> Optional[str]: """The first async offender in this pipeline, or None. Cached. @@ -315,7 +348,7 @@ def _first_async_offender(self, nested: bool = True) -> Optional[str]: for name, step, step_type in self._steps: if step_type == "async_tap": return name - if step_type == "async_filter" and not self._has_sync_path(step): + if step_type == "async_filter" or (step_type == "valve" and not self._has_sync_path(step)): return name if step_type == "pipeline" and nested: inner = step._first_async_offender() @@ -326,9 +359,8 @@ def _first_async_offender(self, nested: bool = True) -> Optional[str]: def _has_sync_path(self, step: Any) -> bool: """True for an async-declared step that can run with no event loop. - ``Valve.call`` is async because its inner filter *may* be, so every - Valve registers as ``async_filter``. A Valve over a sync inner offers - ``call_sync``, which never touches a loop. A Valve over an async (or + ``Valve.call`` is async because its inner filter *may* be. A Valve + over a sync inner can run with no loop. A Valve over an async (or nested Valve) inner is async work. """ if not callable(getattr(step, "call_sync", None)): @@ -336,8 +368,10 @@ def _has_sync_path(self, step: Any) -> bool: inner = getattr(step, "_inner", None) return inner is None or not self._is_async_method(inner, "call") - def _run_sync(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: - self._state = State() + def _run_sync(self, initial_payload: Payload[TInput], state: Optional[State]) -> Payload[TOutput]: + if state is None: + state = State() + self._state = state # the State of the run that started last payload = initial_payload.copy() # the caller's Payload is not changed # Clock reads cost ~0.1 µs each; only take them when someone reads them. @@ -358,8 +392,8 @@ def _run_sync(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: _failing = None if step_type == "tap": - self._observe(name, step, payload) - self._state.mark_executed(name) + self._observe(name, step, payload, state) + state.mark_executed(name) continue if timed: @@ -370,27 +404,29 @@ def _run_sync(self, initial_payload: Payload[TInput]) -> Payload[TOutput]: hook.before(name, payload) if step_type == "pipeline": payload = step.run(payload) - self._state.mark_executed(name) - else: - if step_type == "async_filter": - payload = step.call_sync(payload) # admitted by _has_sync_path + state.mark_executed(name) + elif step_type == "valve": + # The Pipeline asks the Predicate itself: a flag on the + # Valve would be shared by each run that is in progress. + if step._predicate(payload): + payload = step._sync_inner(payload) # admitted by _has_sync_path + state.mark_executed(name) else: - payload = step.call(payload) # type: ignore[attr-defined] - if getattr(step, "_last_skipped", False): - self._state.mark_skipped(name) - else: - self._state.mark_executed(name) + state.mark_skipped(name) + else: + payload = step.call(payload) # type: ignore[attr-defined] + state.mark_executed(name) for hook in hooks: hook.after(name, payload) if timed: - self._state.record_timing(name, time.monotonic() - _step_t0) + state.record_timing(name, time.monotonic() - _step_t0) if self._observe_lineage: - payload = payload._stamp(name) + self._stamp(payload, name) except Exception as e: if timed and _step_t0 is not None and _step_name is not None: - self._state.record_timing(_step_name, time.monotonic() - _step_t0) + state.record_timing(_step_name, time.monotonic() - _step_t0) if _failing is not None: # The failing step first, then the pipeline: on_error mirrors # before/after, which fire at both levels. diff --git a/packages/codeupipe/codeupipe/core/state.py b/packages/codeupipe/codeupipe/core/state.py index 08f1256..0ba7967 100644 --- a/packages/codeupipe/codeupipe/core/state.py +++ b/packages/codeupipe/codeupipe/core/state.py @@ -13,10 +13,10 @@ class State: """The record of one run. A run starts with a new State. - - ``executed``: step identities, in order + - ``executed``: step labels, in order - ``skipped``: steps a Valve did not run - - ``errors``: (identity, error) pairs - - ``timings``: identity -> seconds, when timing is on + - ``errors``: (label, error) pairs + - ``timings``: label -> seconds, when timing is on """ def __init__(self): diff --git a/packages/codeupipe/codeupipe/core/step_identity.py b/packages/codeupipe/codeupipe/core/step_identity.py deleted file mode 100644 index 5ccdc92..0000000 --- a/packages/codeupipe/codeupipe/core/step_identity.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Derive a step's identity from its code (conformance/API.md, "Step identity"). - -A step is identified by the fully qualified symbol of the code that runs it: -``module.QualName`` of a filter's or tap's class, of a Valve's inner filter, -of a named Pipeline type, or of a module-level function. Nothing about it is -written by hand, so it cannot drift from the code it names. -""" - -import sys -import types -from typing import Any - -from .anonymous_step import AnonymousStep - -__all__ = ["identify", "symbol_of", "predicate_symbol", "add_site"] - -_FIX = "Define it at module level (a named class or a module-level def) so it has a stable identity and can be tested on its own." - - -def symbol_of(target: Any, what: str) -> str: - """``module.QualName`` of a class or function, or AnonymousStep.""" - module = getattr(target, "__module__", None) - qualname = getattr(target, "__qualname__", None) - if not module or not qualname: - raise AnonymousStep(f"{what} {target!r} has no module and qualified name. {_FIX}") - if "" in qualname: - raise AnonymousStep(f"{what} is a lambda ({module}.{qualname}). {_FIX}") - if "" in qualname: - raise AnonymousStep(f"{what} {module}.{qualname} is defined inside a function. {_FIX}") - if module == "__main__": - raise AnonymousStep( - f"{what} {qualname} is defined in the entry script (__main__), so its identity changes with " - f"how the program is started and tests cannot import it. Move it into a module.") - return f"{module}.{qualname}" - - -def predicate_symbol(predicate: Any) -> str: - """The identity of a Valve predicate: an object with ``test(payload) -> bool``. - - Predicates are objects (conformance/API.md, "Valve"), identified by - their class, with any configuration on the instance. A function (named, - lambda or closure) is not a predicate object. - """ - if isinstance(predicate, (types.FunctionType, types.BuiltinFunctionType, types.MethodType, type)) \ - or not callable(getattr(predicate, "test", None)): - what = getattr(predicate, "__qualname__", None) or type(predicate).__qualname__ - raise AnonymousStep( - f"Valve predicate {what} is not a predicate object. Declare a module-level class with " - f"test(self, payload) -> bool (configuration on the instance) and pass an instance.") - return symbol_of(type(predicate), "Valve predicate") - - -def identify(step: Any, kind: str) -> str: - """The identity of a step, before any ``#n`` occurrence number. - - kind: "filter", "tap" or "pipeline". - """ - from .pipeline import Pipeline # local: pipeline imports this module - from .valve import Valve - - if kind == "pipeline" or isinstance(step, Pipeline): - if type(step) is Pipeline: - raise AnonymousStep( - "A nested pipeline must be a named Pipeline type (class UserOnboarding(Pipeline): ...), " - "identified like a filter. A plain Pipeline() can only run at the top level.") - return symbol_of(type(step), "Nested pipeline") - if isinstance(step, Valve): - return identify(step._inner, "filter") - if isinstance(step, (type, types.FunctionType, types.BuiltinFunctionType, types.MethodType)): - return symbol_of(step, kind.capitalize()) - return symbol_of(type(step), kind.capitalize()) - - -def add_site(depth: int = 2) -> str: - """``file:line`` of the caller ``depth`` frames up: where the step was added.""" - frame = sys._getframe(depth) - return f"{frame.f_code.co_filename}:{frame.f_lineno}" diff --git a/packages/codeupipe/codeupipe/core/step_label.py b/packages/codeupipe/codeupipe/core/step_label.py new file mode 100644 index 0000000..9195c10 --- /dev/null +++ b/packages/codeupipe/codeupipe/core/step_label.py @@ -0,0 +1,20 @@ +"""The label of a step: the name of its class. + +State and Hooks report a step by this label. It comes from the code, so it +cannot differ from the code. A Valve has the label of its Filter. +""" + +from typing import Any + +__all__ = ["label_of"] + + +# cup: allow-plain a part of the Pipeline primitive: it gives a step its label +def label_of(step: Any) -> str: + """The class name of ``step``; for a Valve, of its Filter.""" + inner = getattr(step, "_inner", None) + if inner is not None and hasattr(step, "_predicate"): # a Valve + return label_of(inner) + if isinstance(step, type): + return step.__name__ + return type(step).__name__ diff --git a/packages/codeupipe/codeupipe/core/valve.py b/packages/codeupipe/codeupipe/core/valve.py index 595e0fc..5817642 100644 --- a/packages/codeupipe/codeupipe/core/valve.py +++ b/packages/codeupipe/codeupipe/core/valve.py @@ -58,11 +58,8 @@ def __init__( inner: Filter[TInput, TOutput], predicate: "Predicate", ): - # A Valve is identified by its inner filter (conformance/API.md); its - # predicate is an object with test(payload) -> bool, identified by its - # class, configuration on the instance. Functions are not predicates. - from .step_identity import predicate_symbol - self.predicate_symbol = predicate_symbol(predicate) + # The predicate is an object with test(payload) -> bool. Its settings + # are on the instance. self._inner = inner self._predicate_object = predicate self._predicate = predicate.test # bound once: called per run @@ -72,6 +69,10 @@ def __init__( self._inner_sync_call = inner_sync if callable(inner_sync) else None self._inner_is_sync = self._inner_sync_call is not None or not inspect.iscoroutinefunction( getattr(inner, "call", None)) + # What a synchronous run calls when the Predicate holds. The Pipeline + # asks the Predicate and calls this itself, and reads no flag from + # the Valve: a flag would be shared by each run that is in progress. + self._sync_inner = self._inner_sync_call if self._inner_sync_call is not None else inner.call async def call(self, payload: Payload[TInput]) -> Payload[TOutput]: """Execute the inner filter only if the predicate passes.""" @@ -111,4 +112,4 @@ def call_sync(self, payload: Payload[TInput]) -> Payload[TOutput]: return result def __repr__(self) -> str: - return f"Valve({type(self._inner).__qualname__}, {self.predicate_symbol})" + return f"Valve({type(self._predicate_object).__name__}, {type(self._inner).__name__})" diff --git a/packages/codeupipe/codeupipe/lower/filters/emit_inline_module_filter.py b/packages/codeupipe/codeupipe/lower/filters/emit_inline_module_filter.py deleted file mode 100644 index f21f4fa..0000000 --- a/packages/codeupipe/codeupipe/lower/filters/emit_inline_module_filter.py +++ /dev/null @@ -1,324 +0,0 @@ -""" -EmitInlineModuleFilter — the whole pipeline as one flattened ``run()``. - -Direction: export (CUP -> standard Python), ``lowering="inline"``. - -The fastest static mode: steps unrolled and inlined, no framework allocations. -Measured on the spike's P1 before building this (µs per run): - - current export (call + dict copy per step) 1.021 - calls kept, one copy + in-place writes 0.849 - inlined, copies kept 0.941 - inlined + one copy + in-place writes 0.723 (hand-written: 0.728) - -Only removing both costs reaches hand-written speed, so this does both: -``run()`` copies the caller's dict once, then each step's body runs inline -and writes keys in place. - -A Payload write is in place, so a step's body is already in-place writes on -``data`` and can stand in ``run()`` as it is. A step is emitted as a function -and called instead when its shape cannot be pasted into ``run()``: an early -return, a nested function that closes over ``data``, a generator, or a step -that rebinds ``data``. -""" - -from __future__ import annotations - -import ast -import copy -import os -from typing import Dict, List, Set - -from codeupipe.core import Payload - -from ..errors import UnsupportedFilterShape - - -__all__ = ["EmitInlineModuleFilter"] - - -class EmitInlineModuleFilter: - """Emit ``//pipeline.py`` holding one flattened ``run``.""" - - def call(self, payload: Payload) -> Payload: - pipeline_ir = payload.get("pipeline_ir") - config = payload.get("config") or {"output": {"base": "src/", "step": "steps/"}} - output = config.get("output", {}) - package_dir = os.path.join(output.get("base", "src/"), output.get("step", "steps/")) - - imports: Dict[str, str] = {} - definitions: List[str] = [] - functions: List[str] = [] - body: List[str] = ["data = dict(data)"] - - for step in pipeline_ir.steps: - prefix = f"_{step.ident}__" - renames = _definition_renames(step, prefix) - helpers: Dict[str, ast.FunctionDef] = {} - for line in step.imports: - _merge_import(imports, line, step) - for source in step.definitions: - tree = ast.parse(source) - _rename_names(tree, renames) - helpers.update({n.name: n for n in tree.body if isinstance(n, ast.FunctionDef)}) - definitions.append(ast.unparse(tree)) - - call_ast = _collapse_double_dict(_renamed_copy(step.call_ast, renames)) - if step.kind == "tap": - body.extend(_tap_lines(step, call_ast, prefix, functions)) - elif step.kind == "valve": - predicate = (_collapse_double_dict(_renamed_copy(step.predicate_ast, renames)) - if step.predicate_ast else None) - if predicate is None: - raise UnsupportedFilterShape( - filter_name=step.name, method_name="predicate", line=0, col=0, - node_kind="FunctionDef", suggestion="Valve reached the emitter without a predicate.", - ) - condition = _predicate_condition(predicate, prefix, functions, helpers) - inner = _filter_lines(step, call_ast, prefix, functions) - body.append(f"if {condition}:") - body.extend(_indent(inner or ["pass"])) - else: - body.extend(_filter_lines(step, call_ast, prefix, functions)) - - body.append("return data") - sections = [ - "\n".join(sorted(imports.values())), - "\n\n".join(definitions), - "\n\n\n".join(functions), - "def run(data):\n" + "\n".join(_indent(body)), - ] - module = "\n\n\n".join(section for section in sections if section) + "\n" - - files = list(payload.get("files") or []) - files.append({"path": os.path.join(package_dir, "pipeline.py"), "content": module}) - files.append({"path": os.path.join(package_dir, "__init__.py"), - "content": "from .pipeline import run\n\n__all__ = [\"run\"]\n"}) - return payload.insert("files", files) - - -# -------------------------------------------------------------------------- -# steps -# -------------------------------------------------------------------------- - -def _filter_lines(step, fn: ast.FunctionDef, prefix: str, functions: List[str]) -> List[str]: - """Inline a filter body, or call it as a function.""" - if _inlinable(fn, allow_final_return=True): - return _inline_filter(fn, prefix) - functions.append(_as_function(fn, f"{prefix}step")) - return [f"data = {prefix}step(data)"] - - -def _tap_lines(step, fn: ast.FunctionDef, prefix: str, functions: List[str]) -> List[str]: - """Plane B: a Tap failure never halts the run (Exception only).""" - if _inlinable(fn, allow_final_return=False): - inner = [ast.unparse(stmt) for stmt in _body_without_docstring(_rename_locals(fn, prefix))] - else: - functions.append(_as_function(fn, f"{prefix}tap")) - inner = [f"{prefix}tap(data)"] - return ["try:", *_indent(inner or ["pass"]), "except Exception:", " pass"] - - -def _indent(statements: List[str]) -> List[str]: - """Indent every line; an unparsed compound statement spans several.""" - return [" " + line for stmt in statements for line in stmt.splitlines()] - - -def _predicate_condition(fn: ast.FunctionDef, prefix: str, functions: List[str], - helpers: Dict[str, ast.FunctionDef]) -> str: - body = _body_without_docstring(fn) - if len(body) == 1 and isinstance(body[0], ast.Return) and body[0].value is not None: - return ast.unparse(_inline_helper_call(body[0].value, helpers)) - functions.append(_as_function(fn, f"{prefix}predicate")) - return f"{prefix}predicate(data)" - - -def _inline_filter(fn: ast.FunctionDef, prefix: str) -> List[str]: - fn = _rename_locals(fn, prefix) - statements = _body_without_docstring(fn) - *head, last = statements - lines = [ast.unparse(stmt) for stmt in head] - value = last.value - if isinstance(value, ast.Name) and value.id == "data": - return lines - return lines + [f"data = {ast.unparse(value)}"] - - -# -------------------------------------------------------------------------- -# redundant copies -# -------------------------------------------------------------------------- - -class _CollapseDoubleDict(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - node = self.generic_visit(node) - if _is_dict_call(node) and _is_dict_call(node.args[0]): - return node.args[0] - return node - - -def _is_dict_call(node: ast.AST) -> bool: - return (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "dict" - and len(node.args) == 1 and not node.keywords) - - -def _collapse_double_dict(fn: ast.AST) -> ast.AST: - """``dict(dict(x))`` is ``dict(x)``.""" - return ast.fix_missing_locations(_CollapseDoubleDict().visit(fn)) - - -def _inline_helper_call(expr: ast.AST, helpers: Dict[str, ast.FunctionDef]) -> ast.AST: - """``helper(dict(data))`` -> the helper's return expression over ``data``, - when the helper is ``def helper(d): return `` and only reads ``d`` - through ``d.get(...)``. Generated adapters wrap Valve predicates this - way, copying the whole dict on each check.""" - if not (isinstance(expr, ast.Call) and isinstance(expr.func, ast.Name) and len(expr.args) == 1 - and not expr.keywords and _is_dict_call(expr.args[0]) - and isinstance(expr.args[0].args[0], ast.Name) and expr.args[0].args[0].id == "data"): - return expr - helper = helpers.get(expr.func.id) - if helper is None or len(helper.args.args) != 1 or helper.args.vararg or helper.args.kwarg: - return expr - body = _body_without_docstring(helper) - if not (len(body) == 1 and isinstance(body[0], ast.Return) and body[0].value is not None): - return expr - param = helper.args.args[0].arg - returned = copy.deepcopy(body[0].value) - parents = {child: parent for parent in ast.walk(returned) for child in ast.iter_child_nodes(parent)} - for node in ast.walk(returned): - if isinstance(node, ast.Name) and node.id == param: - parent = parents.get(node) - if not (isinstance(parent, ast.Attribute) and parent.attr == "get" - and isinstance(parents.get(parent), ast.Call)): - return expr - node.id = "data" - elif isinstance(node, (ast.Lambda, ast.NamedExpr)): - return expr - return returned - - -# -------------------------------------------------------------------------- -# inlinability: can `data` escape this body? -# -------------------------------------------------------------------------- - -def _inlinable(fn: ast.FunctionDef, allow_final_return: bool, any_return: bool = False) -> bool: - body = _body_without_docstring(fn) - if not body: - return not allow_final_return - for node in ast.walk(fn): - if isinstance(node, (ast.Yield, ast.YieldFrom, ast.Await, ast.Global, ast.Nonlocal)): - return False - if isinstance(node, (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: - if any(isinstance(n, ast.Name) and n.id == "data" for n in ast.walk(node)): - return False - if isinstance(node, ast.Name) and node.id == "data" and not isinstance(node.ctx, ast.Load): - return False - returns = [n for n in ast.walk(fn) if isinstance(n, ast.Return)] - final = body[-1] if isinstance(body[-1], ast.Return) else None - if any_return: - pass - elif allow_final_return: - if final is None or final.value is None or returns != [final]: - return False - elif returns: - return False - return True - - -# -------------------------------------------------------------------------- -# names -# -------------------------------------------------------------------------- - -def _definition_renames(step, prefix: str) -> Dict[str, str]: - renames: Dict[str, str] = {} - for source in step.definitions: - for stmt in ast.parse(source).body: - for name in _top_level_bound(stmt): - renames[name] = prefix + name - return renames - - -def _top_level_bound(stmt: ast.stmt) -> Set[str]: - if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - return {stmt.name} - names: Set[str] = set() - targets = stmt.targets if isinstance(stmt, ast.Assign) else [getattr(stmt, "target", None)] - for target in targets: - for node in ast.walk(target) if target is not None else (): - if isinstance(node, ast.Name): - names.add(node.id) - return names - - -def _rename_names(tree: ast.AST, renames: Dict[str, str]) -> None: - for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id in renames: - node.id = renames[node.id] - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.name in renames: - node.name = renames[node.name] - - -def _renamed_copy(fn: ast.AST, renames: Dict[str, str]) -> ast.AST: - fn = copy.deepcopy(fn) - _rename_names(fn, renames) - return fn - - -def _rename_locals(fn: ast.FunctionDef, prefix: str) -> ast.FunctionDef: - """Prefix every name the body binds, so inlined steps cannot collide.""" - fn = copy.deepcopy(fn) - local: Set[str] = set() - for node in ast.walk(fn): - if isinstance(node, ast.Name) and not isinstance(node.ctx, ast.Load): - local.add(node.id) - elif isinstance(node, ast.arg) and node is not fn.args.args[0]: - local.add(node.arg) - elif isinstance(node, ast.ExceptHandler) and node.name: - local.add(node.name) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: - local.add(node.name) - local.discard("data") - renames = {name: prefix + name for name in local} - for node in ast.walk(fn): - if isinstance(node, ast.Name) and node.id in renames: - node.id = renames[node.id] - elif isinstance(node, ast.arg) and node.arg in renames: - node.arg = renames[node.arg] - elif isinstance(node, ast.ExceptHandler) and node.name in renames: - node.name = renames[node.name] - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: - node.name = renames.get(node.name, node.name) - return fn - - -def _body_without_docstring(fn: ast.FunctionDef) -> List[ast.stmt]: - body = list(fn.body) - if body and isinstance(body[0], ast.Expr) and isinstance(getattr(body[0], "value", None), ast.Constant) \ - and isinstance(body[0].value.value, str): - body = body[1:] - return body - - -def _as_function(fn: ast.FunctionDef, name: str) -> str: - fn = copy.deepcopy(fn) - fn.name = name - return ast.unparse(fn) - - -def _merge_import(imports: Dict[str, str], line: str, step) -> None: - bound = _top_level_bound_import(line) - for name in bound: - existing = imports.get(name) - if existing is not None and existing != line: - raise UnsupportedFilterShape( - filter_name=step.name, method_name="imports", line=0, col=0, node_kind="Import", - suggestion=f"`{name}` is imported differently by two steps ({existing!r} vs {line!r}); " - "use the default per-step lowering for this pipeline.", - ) - imports[name] = line - - -def _top_level_bound_import(line: str) -> Set[str]: - stmt = ast.parse(line).body[0] - if isinstance(stmt, ast.Import): - return {alias.asname or alias.name.split(".")[0] for alias in stmt.names} - return {alias.asname or alias.name for alias in stmt.names} diff --git a/packages/codeupipe/codeupipe/lower/filters/emit_orchestrator_filter.py b/packages/codeupipe/codeupipe/lower/filters/emit_orchestrator_filter.py deleted file mode 100644 index 74f2e74..0000000 --- a/packages/codeupipe/codeupipe/lower/filters/emit_orchestrator_filter.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -EmitOrchestratorFilter — the ``def run(data): ...`` glue. - -Direction: export (CUP -> standard Python). -Milestone: M1. - -Two forms, chosen by the export payload's ``observe`` flag: - -* fast (default): ``run(data)`` calls each step in order. Same output as - the dynamic run, no State, no Hooks. -* observable (``observe=True``): ``run(data, state=None, hooks=None, - timing=False, trace_id=None)`` also records State into the ``state`` dict - and calls Hooks, in exactly the order the dynamic ``Pipeline.run`` does - (conformance/API.md, "Hook" and "State"). A nested pipeline's steps are - run inline but reported as the one step that nested them. -""" - -from __future__ import annotations - -import os -from itertools import groupby -from typing import List - -from codeupipe.core import Payload - - -__all__ = ["EmitOrchestratorFilter"] - -def _default_config(payload: Payload): - return payload.get("config") or { - "pattern": payload.get("pattern", "flat"), - "output": {"base": "src/", "step": "steps/"}, - } - - -def _import_lines(steps) -> List[str]: - # Each step lives in its own sibling module, so the orchestrator must - # import it. Relative imports keep step names (``json``, ``truncate``) - # from shadowing top-level modules on the customer's sys.path. - # Code uses each step's generated identifier (``step_03_validate``), never - # its identity: identities are labels, reported in State and to Hooks. - lines = [] - for step in steps: - names = [step.ident] - if step.kind == "valve": - names.append(f"{step.ident}_predicate") - lines.append(f"from .{step.ident} import {', '.join(names)}") - return lines - - -def _plain_lines(step, pad: str, var: str = "data") -> List[str]: - """One step, no bookkeeping: the fast form, and a nested pipeline's body - (which works on its own variable, ``var``).""" - fn = step.ident - if step.kind == "valve": - return [f"{pad}if {fn}_predicate({var}):", f"{pad} {var} = {fn}({var})"] - if step.kind == "tap": - # Plane B: a Tap failure never halts the run, matching the dynamic - # runtime. Exception only; KeyboardInterrupt etc. pass. - return [f"{pad}try:", f"{pad} {fn}({var})", f"{pad}except Exception:", f"{pad} pass"] - return [f"{pad}{var} = {fn}({var})"] - - -def _fast_run(steps) -> List[str]: - # Steps write in place, so run() works on its own copy of the caller's dict. - lines = ["def run(data):", " data = dict(data)"] - for step in steps: - lines.extend(_plain_lines(step, " ")) - lines.append(" return data") - return lines - - -def _observed_step(name: str, body: List[str], pad: str) -> List[str]: - """A Plane A step: hooks around it, timing, the failing-step marker.""" - return [ - f"{pad}_step = {name!r}", - f"{pad}if timing:", - f"{pad} _t0 = _clock()", - f"{pad}for _hook in hooks:", - f"{pad} _hook.before({name!r}, data)", - *body, - f"{pad}for _hook in hooks:", - f"{pad} _hook.after({name!r}, data)", - f"{pad}if timing:", - f"{pad} timings[{name!r}] = _clock() - _t0", - ] - - -def _observed_run(steps, source_hooks) -> List[str]: - pad = " " - body: List[str] = [] - for group, members in groupby(steps, key=lambda s: s.group): - members = list(members) - if group is not None: - # A nested pipeline: one step in this run's State and Hooks. Its - # own skips and Tap errors belong to its run, as dynamically. - # It works on its own copy, as the nested run does: if it fails, the outer on_error hooks see ``data`` as - # it was when the nested pipeline was entered, as dynamically. - inner = [line for step in members for line in _plain_lines(step, pad, "_nested")] - body.extend(_observed_step(group, [f"{pad}_nested = dict(data)", *inner, f"{pad}data = _nested", - f"{pad}executed.append({group!r})"], pad)) - continue - for step in members: - name, fn = step.name, step.ident - if step.kind == "tap": - body.extend([ - f"{pad}_step = None", - f"{pad}try:", - f"{pad} {fn}(data)", - f"{pad}except Exception as _error:", - f"{pad} errors.append(({name!r}, _error))", - f"{pad}executed.append({name!r})", - ]) - elif step.kind == "valve": - body.extend(_observed_step(name, [ - f"{pad}if {fn}_predicate(data):", - f"{pad} data = {fn}(data)", - f"{pad} executed.append({name!r})", - f"{pad}else:", - f"{pad} skipped.append({name!r})", - ], pad)) - else: - body.extend(_observed_step(name, [ - f"{pad}data = {fn}(data)", - f"{pad}executed.append({name!r})", - ], pad)) - return [ - "from time import perf_counter as _clock", - "", - f"SOURCE_HOOKS = {tuple(source_hooks)!r}", - "", - "", - "def run(data, state=None, hooks=None, timing=False, trace_id=None):", - ' """Run the pipeline, recording State into ``state`` and calling ``hooks``.', - "", - " state: a dict to fill with executed, skipped, errors [(name, error)],", - " timings {name: seconds} (only when ``timing``) and trace_id.", - " hooks: objects with before(step, data), after(step, data) and", - " on_error(step, error, data); step is None for the pipeline itself.", - ' """', - " if hooks is None:", - " if SOURCE_HOOKS:", - " raise TypeError(", - ' "this pipeline was exported with Hooks (" + ", ".join(SOURCE_HOOKS) + "): "', - ' "pass them as run(..., hooks=[...]), or hooks=() to run without them")', - " hooks = ()", - " if state is None:", - " state = {}", - ' executed = state["executed"] = []', - ' skipped = state["skipped"] = []', - ' errors = state["errors"] = []', - ' timings = state["timings"] = {}', - ' state["trace_id"] = trace_id', - " _step = None", - " _t0 = 0.0", - " data = dict(data)", - " for _hook in hooks:", - " _hook.before(None, data)", - " try:", - *(body or [f"{pad}pass"]), - " except Exception as _error:", - " if _step is not None:", - " if timing:", - " timings[_step] = _clock() - _t0", - " for _hook in hooks:", - " _hook.on_error(_step, _error, data)", - " for _hook in hooks:", - " _hook.on_error(None, _error, data)", - " raise", - " for _hook in hooks:", - " _hook.after(None, data)", - " return data", - ] - - -class EmitOrchestratorFilter: - """Emit the top-level orchestrator that calls each step in order.""" - - def call(self, payload: Payload) -> Payload: - pipeline_ir = payload.get("pipeline_ir") - config = _default_config(payload) - output = config.get("output", {}) - base = output.get("base", "src/") - step_dir = output.get("step", "steps/") - files = list(payload.get("files") or []) - - lines = _import_lines(pipeline_ir.steps) - if payload.get("observe"): - lines.extend(["", ""] + _observed_run(pipeline_ir.steps, payload.get("source_hooks") or ())) - else: - lines.extend(["", ""] + _fast_run(pipeline_ir.steps)) - lines.append("") - - files.append({ - "path": os.path.join(base, step_dir, "pipeline.py"), - "content": "\n".join(lines), - }) - files.append({ - "path": os.path.join(base, step_dir, "__init__.py"), - "content": "from .pipeline import run\n\n__all__ = [\"run\"]\n", - }) - return payload.insert("files", files) diff --git a/packages/codeupipe/codeupipe/lower/filters/parse_filter_source_filter.py b/packages/codeupipe/codeupipe/lower/filters/parse_filter_source_filter.py deleted file mode 100644 index 03d817f..0000000 --- a/packages/codeupipe/codeupipe/lower/filters/parse_filter_source_filter.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -ParseFilterSourceFilter — extract each Filter's ``call`` method as an AST. - -Direction: export (CUP -> standard Python). -Milestone: M0. -""" - -from __future__ import annotations - -import ast -import builtins -import inspect -import sys -import textwrap -import types -from dataclasses import replace -from typing import Iterable, Optional, Tuple - -from codeupipe.core import Payload - -from ..errors import UnsupportedFilterShape - - -__all__ = ["ParseFilterSourceFilter"] - - -_BUILTIN_NAMES = frozenset(dir(builtins)) - - -class ParseFilterSourceFilter: - """Turn each step's live instance into an ``ast.FunctionDef``.""" - - def call(self, payload: Payload) -> Payload: - pipeline_ir = payload.get("pipeline_ir") - step_instances = payload.get("step_instances") or [] - step_predicates = payload.get("step_predicates") or [None] * len(step_instances) - rewritten_steps = [] - - for step, instance, predicate in zip(pipeline_ir.steps, step_instances, step_predicates): - method_name = "observe" if step.kind == "tap" else "call" - method = getattr(instance, method_name, None) - if method is None: - raise UnsupportedFilterShape( - filter_name=step.name, - method_name=method_name, - line=getattr(step.call_ast, "lineno", 0), - col=getattr(step.call_ast, "col_offset", 0), - node_kind="Attribute", - suggestion="Define the step in a real .py file with a concrete call/observe method.", - ) - try: - source = textwrap.dedent(inspect.getsource(method)) - except (OSError, TypeError): - raise UnsupportedFilterShape( - filter_name=step.name, - method_name=method_name, - line=getattr(step.call_ast, "lineno", 0), - col=getattr(step.call_ast, "col_offset", 0), - node_kind="FunctionDef", - suggestion="Define the step in a real .py file so inspect.getsource can recover it.", - ) - function_ast = _erase_annotations(ast.parse(source).body[0]) - function_ast, config = _bind_instance_config(function_ast, instance, step) - imports, definitions = _detect_dependencies(function_ast, method, step.name) - definitions = config + definitions - predicate_ast = None - if step.kind == "valve": - predicate_ast, p_config = _parse_predicate(step, predicate) - p_imports, p_definitions = _detect_dependencies(predicate_ast, predicate.test, step.name) - imports = tuple(sorted(set(imports) | set(p_imports))) - p_definitions = p_config + p_definitions - definitions = definitions + tuple(d for d in p_definitions if d not in definitions) - rewritten_steps.append( - replace( - step, - call_ast=function_ast, - docstring=ast.get_docstring(function_ast), - imports=imports, - definitions=definitions, - predicate_ast=predicate_ast, - ) - ) - - return payload.insert("pipeline_ir", replace(pipeline_ir, steps=rewritten_steps)) - - -class _EraseAnnotations(ast.NodeTransformer): - """Drop every type hint in a function: on its parameters, its return, and - its locals (``x: T = v`` becomes ``x = v``; a bare ``x: T`` goes).""" - - def _function(self, node): - node.returns = None - arguments = node.args - for argument in arguments.posonlyargs + arguments.args + arguments.kwonlyargs: - argument.annotation = None - for argument in (arguments.vararg, arguments.kwarg): - if argument is not None: - argument.annotation = None - return self.generic_visit(node) - - visit_FunctionDef = visit_AsyncFunctionDef = _function - - def generic_visit(self, node): - node = super().generic_visit(node) - # A block whose only statement was a bare hint must not be left empty. - if isinstance(getattr(node, "body", None), list) and not node.body: - node.body = [ast.Pass()] - return node - - def visit_AnnAssign(self, node): - if node.value is None: - return None - return ast.copy_location(ast.Assign(targets=[node.target], value=self.visit(node.value)), node) - - -def _erase_annotations(fn): - """A step's type hints say what it reads and writes; they are for the - author and the tools, as in TypeScript. Exported code carries none, so it - never refers to the hint's names (``Payload``, a TypedDict) and so never - imports the framework.""" - return ast.fix_missing_locations(_EraseAnnotations().visit(fn)) - - -def _bind_instance_config(fn: ast.FunctionDef, instance, step, prefix: str = "_self_") -> Tuple[ast.FunctionDef, tuple]: - """Compile a step's constructor config into it as named constants. - - Export lowers a *live* Pipeline, so each ``self.`` the method reads - already has its value. A value with a literal form is baked in as - ``_self_ = `` — per step, so two instances of one class - keep their own config. Anything that cannot be compiled in is refused - here rather than left as a reference to a ``self`` that will not exist. - """ - if not fn.args.args: - return fn, () - self_name = fn.args.args[0].arg - constants: dict = {} - - def refuse(attr: str, why: str, node: ast.AST) -> UnsupportedFilterShape: - return UnsupportedFilterShape( - filter_name=step.name, - method_name=fn.name, - line=getattr(node, "lineno", 0), - col=getattr(node, "col_offset", 0), - node_kind="Attribute", - suggestion=f"`{self_name}.{attr}` {why}", - ) - - class _Bind(ast.NodeTransformer): - def visit_Attribute(self, node: ast.Attribute): - if not (isinstance(node.value, ast.Name) and node.value.id == self_name): - return self.generic_visit(node) - attr = node.attr - if not isinstance(node.ctx, ast.Load): - raise refuse(attr, "is written inside the step; per-instance state has no static form.", node) - if attr not in constants: - if not hasattr(instance, attr): - raise refuse(attr, "is not set on the instance being exported.", node) - value = getattr(instance, attr) - if callable(value): - raise refuse(attr, "is a method or callable; inline it, or make it a module-level function.", node) - literal = _literal_source(value) - if literal is None: - raise refuse(attr, f"holds a {type(value).__name__}, which has no literal form to compile in.", node) - constants[attr] = (f"{prefix}{attr}", literal) - return ast.copy_location(ast.Name(id=constants[attr][0], ctx=ast.Load()), node) - - def visit_Name(self, node: ast.Name): - if node.id == self_name: - raise refuse("", f"`{self_name}` is used directly; only reads of literal attributes can be compiled in.", node) - return node - - body = [_Bind().visit(stmt) for stmt in fn.body] - fn.body = [ast.fix_missing_locations(stmt) for stmt in body] - definitions = tuple(f"{name} = {literal}" for name, literal in constants.values()) - return fn, definitions - - -def _literal_source(value) -> Optional[str]: - """``repr(value)`` if it evaluates back to an equal value of the same type.""" - source = repr(value) - try: - back = ast.literal_eval(source) - except (ValueError, SyntaxError, TypeError, MemoryError, RecursionError): - return None - if type(back) is not type(value) or back != value: - return None - return source - - -def _parse_predicate(step, predicate) -> Tuple[ast.FunctionDef, tuple]: - """Recover a Valve predicate object's ``test`` as a ``def`` taking ``payload``. - - Predicates are objects (conformance/API.md, "Valve"): the ``test`` - method is lowered like a filter's ``call``, the instance's literal - settings compiled in as constants (``_when_``, so they cannot - collide with the filter's ``_self_`` in the same module). The - orchestrator always emits ``if _predicate(data):``, so a predicate - that cannot be recovered stops the export here. - """ - def refuse(suggestion: str, kind: str = "FunctionDef") -> UnsupportedFilterShape: - return UnsupportedFilterShape( - filter_name=step.name, method_name="predicate", line=0, col=0, node_kind=kind, suggestion=suggestion, - ) - - method = getattr(predicate, "test", None) - if method is None: - raise refuse("A Valve predicate is an object with test(self, payload) -> bool.") - try: - fn = _erase_annotations(ast.parse(textwrap.dedent(inspect.getsource(method))).body[0]) - except (OSError, TypeError, SyntaxError, IndexError): - raise refuse("Define the predicate class in a real .py file so inspect.getsource can recover it.") - if not isinstance(fn, ast.FunctionDef) or len(fn.args.args) != 2: - raise refuse("A Valve predicate's test must be a plain `def test(self, payload)`.") - fn, config = _bind_instance_config(fn, predicate, step, prefix="_when_") - fn.args.args = fn.args.args[1:] # self is gone: its reads are now constants - return _rename_parameter(fn, fn.args.args[0].arg, "payload"), config - - -def _rename_parameter(fn: ast.FunctionDef, old: str, new: str) -> ast.FunctionDef: - """Rename the predicate's parameter so RewritePayloadOps recognises it.""" - if old == new: - return fn - fn.args.args[0].arg = new - for node in ast.walk(fn): - if isinstance(node, ast.Name) and node.id == old: - node.id = new - return fn - - -def _detect_dependencies(fn: ast.FunctionDef, obj, step_name: str) -> Tuple[tuple, tuple]: - """Return ``(imports, definitions)`` the step body needs to run standalone. - - Names are resolved against the defining module's *source* first, because - that is what the author wrote: a top-level import is re-emitted as that - import, and a top-level constant, helper, or class is copied verbatim - (with whatever it needs in turn). Only names the source does not bind - fall back to runtime introspection of the value. - - Resolving by value alone is what dropped same-module constants and - turned ``_WORDS = re.compile(...)`` into ``from re import _WORDS``: a - compiled pattern reports ``__module__ == "re"``. A name that neither - route can place raises instead of being dropped. - """ - module = inspect.getmodule(obj) - if module is None: - return (), () - module_ns = getattr(module, "__dict__", {}) - if not module_ns: - return (), () - bindings = _module_bindings(module) - - imports: set = set() - definitions: dict = {} - pending = list(_referenced_free_names(fn, _local_names(fn))) - seen: set = set() - while pending: - name = pending.pop() - if name in seen or name in _BUILTIN_NAMES: - continue - seen.add(name) - - stmt = bindings.get(name) - if isinstance(stmt, (ast.Import, ast.ImportFrom)): - import_line = _import_line_for_alias(stmt, name) - if import_line is not None: - imports.add(import_line) - continue - elif stmt is not None: - definitions[stmt.lineno] = ast.unparse(stmt) - pending.extend(_free_names(stmt)) - continue - - if name not in module_ns: - continue - import_line = _import_statement_for(name, module_ns[name], module.__name__) - if import_line is None: - raise UnsupportedFilterShape( - filter_name=step_name, - method_name=fn.name, - line=getattr(fn, "lineno", 0), - col=getattr(fn, "col_offset", 0), - node_kind="Name", - suggestion=( - f"`{name}` has no top-level definition or import in {module.__name__}'s source " - "and cannot be imported by name; define it at module level so export can copy it." - ), - ) - imports.add(import_line) - - return tuple(sorted(imports)), tuple(definitions[line] for line in sorted(definitions)) - - -def _module_bindings(module) -> dict: - """Map each name to the last top-level statement in the module that binds it.""" - try: - tree = ast.parse(inspect.getsource(module)) - except (OSError, TypeError, SyntaxError): - return {} - bindings: dict = {} - for stmt in tree.body: - if isinstance(stmt, ast.Import): - for alias in stmt.names: - bindings[alias.asname or alias.name.split(".")[0]] = stmt - elif isinstance(stmt, ast.ImportFrom): - for alias in stmt.names: - bindings[alias.asname or alias.name] = stmt - elif isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bindings[stmt.name] = stmt - elif isinstance(stmt, ast.Assign): - for target in stmt.targets: - names: set = set() - _collect_target_names(target, names) - for name in names: - bindings[name] = stmt - elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None: - if isinstance(stmt.target, ast.Name): - bindings[stmt.target.id] = stmt - return bindings - - -def _import_line_for_alias(stmt, name: str) -> Optional[str]: - """Re-emit only the alias that binds ``name``; relative imports return None.""" - if isinstance(stmt, ast.ImportFrom): - if stmt.level or stmt.module == "__future__": - return None - for alias in stmt.names: - if (alias.asname or alias.name) == name: - suffix = f" as {alias.asname}" if alias.asname else "" - return f"from {stmt.module} import {alias.name}{suffix}" - return None - for alias in stmt.names: - if (alias.asname or alias.name.split(".")[0]) == name: - return f"import {alias.name} as {alias.asname}" if alias.asname else f"import {alias.name}" - return None - - -def _free_names(stmt: ast.AST) -> set: - """Names a copied top-level statement loads but does not itself bind.""" - loaded: set = set() - bound: set = set() - for node in ast.walk(stmt): - if isinstance(node, ast.Name): - (loaded if isinstance(node.ctx, ast.Load) else bound).add(node.id) - elif isinstance(node, ast.arg): - bound.add(node.arg) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - bound.add(node.name) - return loaded - bound - - -def _local_names(fn: ast.FunctionDef) -> set: - names = {"self", "cls"} - args = fn.args - for arg in list(args.args) + list(args.posonlyargs) + list(args.kwonlyargs): - names.add(arg.arg) - if args.vararg is not None: - names.add(args.vararg.arg) - if args.kwarg is not None: - names.add(args.kwarg.arg) - for node in ast.walk(fn): - if isinstance(node, ast.Assign): - for target in node.targets: - _collect_target_names(target, names) - elif isinstance(node, ast.AugAssign) or isinstance(node, ast.AnnAssign): - _collect_target_names(node.target, names) - elif isinstance(node, (ast.For, ast.comprehension)): - _collect_target_names(node.target, names) - elif isinstance(node, ast.With): - for item in node.items: - if item.optional_vars is not None: - _collect_target_names(item.optional_vars, names) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node is not fn: - names.add(node.name) - return names - - -def _collect_target_names(target: ast.AST, names: set) -> None: - if isinstance(target, ast.Name): - names.add(target.id) - elif isinstance(target, (ast.Tuple, ast.List)): - for elt in target.elts: - _collect_target_names(elt, names) - elif isinstance(target, ast.Starred): - _collect_target_names(target.value, names) - - -def _referenced_free_names(fn: ast.FunctionDef, local_names: set) -> Iterable[str]: - seen: set = set() - for node in ast.walk(fn): - root = None - if isinstance(node, ast.Name): - root = node.id - elif isinstance(node, ast.Attribute): - root = _leftmost_name(node) - if root and root not in local_names and root not in seen: - seen.add(root) - yield root - - -def _leftmost_name(node: ast.Attribute) -> Optional[str]: - current = node.value - while isinstance(current, ast.Attribute): - current = current.value - if isinstance(current, ast.Name): - return current.id - return None - - -def _import_statement_for(local_name: str, value, current_module: str) -> Optional[str]: - if isinstance(value, types.ModuleType): - real_name = value.__name__ - if real_name == local_name: - return f"import {real_name}" - return f"import {real_name} as {local_name}" - - origin = getattr(value, "__module__", None) - if not origin or origin == current_module or origin == "builtins": - return None - real_name = getattr(value, "__name__", local_name) - # ``__module__`` names where a value's *type* lives, not where the value - # can be imported from (a compiled pattern reports "re"). Only emit the - # import if it would actually hand back this object. - if getattr(sys.modules.get(origin), real_name, None) is not value: - return None - if real_name == local_name: - return f"from {origin} import {local_name}" - return f"from {origin} import {real_name} as {local_name}" diff --git a/packages/codeupipe/pyproject.toml b/packages/codeupipe/pyproject.toml index a6b0291..7d2859b 100644 --- a/packages/codeupipe/pyproject.toml +++ b/packages/codeupipe/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "codeupipe" -version = "1.0.0" +version = "2.0.0b1" description = "A Python library for writing a program as a pipeline of small steps." readme = "README.md" requires-python = ">=3.9" @@ -13,6 +13,7 @@ license-files = ["LICENSE", "NOTICE"] authors = [{name = "Joshua Wink"}] keywords = ["pipeline", "filter", "payload", "composition"] classifiers = [ + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python :: 3", diff --git a/pyproject.toml b/pyproject.toml index a93f0e1..ee6e44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,7 @@ [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] -# The speed check measures time, so it is not in the default run: pytest -m perf -addopts = ["--tb=short", "-m", "not perf"] +addopts = ["--tb=short"] asyncio_mode = "strict" markers = [ "unit: Unit tests", @@ -16,5 +15,4 @@ markers = [ "core: Core functionality tests", "asyncio: Async tests", "slow: Slow tests (skippable during dev with -m \"not slow\")", - "perf: The speed check against the recorded baseline (python conformance/bench.py --check)", ] diff --git a/scorecard/README.md b/scorecard/README.md index 6a191af..92c98f1 100644 --- a/scorecard/README.md +++ b/scorecard/README.md @@ -1,6 +1,6 @@ # Scorecard -The scorecard compares the static export of a Pipeline (`codeupipe.lower`) +The scorecard compares the compiled form of a Pipeline (`codeupipe.compile`) with the dynamic `Pipeline.run` on random pipelines. Each number in this document comes from a command in this document. @@ -29,8 +29,8 @@ steps) that cross the Payload's overlay-flattening depth. `--hostile-rate` generation (builtins, the generated `run()`'s locals, duplicates, non-identifiers). -Each case runs in four modes: dynamic (reference), fast export `steps`, -fast export `inline`, observable export (`observe=True`). +Each case runs in four modes: dynamic (reference), fast mode `steps`, +fast mode `inline`, observable mode (`observe=True`). | check | steps | inline | observe | |---|---|---|---| @@ -43,16 +43,16 @@ fast export `inline`, observable export (`observe=True`). | Hook event sequence, with a deep snapshot of the payload at every event | | | ✓ | | `SOURCE_HOOKS` guard: `run()` without `hooks=` raises `TypeError` | | | ✓ | -**Verdicts.** *pass*: agrees on every check. *refused*: export raised +**Verdicts.** *pass*: agrees on every check. *refused*: the compiler raised `UnsupportedFilterShape`, counted separately, never as a pass. *fail*: any -divergence, or an export that crashed (a `SyntaxError` on load is not a +divergence, or compiled code that crashed (a `SyntaxError` on load is not a refusal). Every distinct failure signature is shrunk (`minimize.py`) to a minimal case and printed. **Isolation (item 4)** is measured inside the same runs: the caller's input is deep-compared after every run in every mode, and a second run on a fresh copy must give the same result. A `SnapshotHook` (`snapshot_hook.py`) is -attached to the dynamic run and passed to the observable export's `run()`; +attached to the dynamic run and passed to the observable mode's `run()`; at each event it deep-copies the payload, because a Payload is written in place and the object changes later by design. diff --git a/scorecard/case_gen.py b/scorecard/case_gen.py index 3302630..176e475 100644 --- a/scorecard/case_gen.py +++ b/scorecard/case_gen.py @@ -46,6 +46,7 @@ ] +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def generate_value(rng, depth=0): """A JSON-shaped value (None, bool, int, float, str, list, dict).""" roll = rng.random() @@ -68,6 +69,7 @@ def _add_operand(rng): return rng.choice([1, 1, 2, -3, 0.5, 10 ** 20, True, "s", [1], None]) +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _filter_spec(rng): op = rng.choices(FILTER_OPS, weights=[5, 3, 4, 3, 3, 2, 2, 2, 1])[0] key = rng.choice(KEYS) @@ -90,12 +92,14 @@ def _filter_spec(rng): return {"op": "fail", "message": rng.choice(["boom", "bad 'input'", "ünïcode"])} +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _when(rng): if rng.random() < 0.5: return {"op": "truthy", "key": rng.choice(KEYS)} return {"op": "equals", "key": rng.choice(KEYS), "value": rng.choice([0, 1, True, None, "x", "", [1], {"a": 1}])} +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _namer(rng, hostile): """next_name(prefix): unique ``prefix``, or sometimes a hostile name.""" counter = [0] @@ -108,6 +112,7 @@ def next_name(prefix): return next_name +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _steps(rng, names, depth, max_steps, max_depth): out = [] for _ in range(rng.randint(0, max_steps)): @@ -131,6 +136,7 @@ def _steps(rng, names, depth, max_steps, max_depth): return out +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def generate_case(rng, index=0, max_steps=8, max_depth=2, hostile_rate=0.0): """One random case. ``rng`` is a ``random.Random``; same seed, same case. diff --git a/scorecard/compare.py b/scorecard/compare.py index c69e35c..071d0d1 100644 --- a/scorecard/compare.py +++ b/scorecard/compare.py @@ -10,12 +10,14 @@ __all__ = ["divergences", "self_checks", "verdict"] +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _diff(check, a, b, out): detail = first_difference(a, b) if detail: out.append({"check": check, "detail": detail}) +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def self_checks(record, who): """Checks a run makes of itself: input untouched, reruns identical.""" out = [] @@ -26,6 +28,7 @@ def self_checks(record, who): return out +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def divergences(dynamic, record, mode): out = [] if record["status"] == "crash": @@ -47,6 +50,7 @@ def divergences(dynamic, record, mode): return out +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def verdict(record, found): if found: return "fail" diff --git a/scorecard/differential.py b/scorecard/differential.py index 68a9a2a..d9a6b31 100644 --- a/scorecard/differential.py +++ b/scorecard/differential.py @@ -31,6 +31,7 @@ RESULTS = pathlib.Path(__file__).resolve().parent / "results" +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def check_case(case): """{"dynamic": [self-check failures], "modes": {mode: {verdict, divergences, reason}}}""" predicates = vocab.make_predicates(case["steps"], vocab.new_workdir()) @@ -44,6 +45,7 @@ def check_case(case): return result +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def failure_signature(result): """What a reproducer must keep failing on: (mode, check) pairs.""" sig = {("dynamic", d["check"]) for d in result["dynamic"]} @@ -52,6 +54,7 @@ def failure_signature(result): return sig +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _refusal_kind(reason): """'' from 'name.call at line 0:0 — cannot lower . Hint: ...'""" if reason and "cannot lower " in reason: @@ -59,6 +62,7 @@ def _refusal_kind(reason): return reason or "?" +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def run_campaign(seed, count, hostile_rate=0.15, max_repros=12, max_steps=8, max_depth=2): started = time.time() cases = case_gen.generate_cases(seed, count, max_steps=max_steps, max_depth=max_depth, @@ -120,6 +124,7 @@ def run_campaign(seed, count, hostile_rate=0.15, max_repros=12, max_steps=8, max } +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def markdown(report): lines = [f"# Differential run: seed {report['seed']}, {report['count']} cases", "", f"{report['seconds']} s. Corpus: " + ", ".join(f"{k} {v}" for k, v in report["corpus"].items()) + ".", @@ -152,6 +157,7 @@ def markdown(report): return "\n".join(lines) + "\n" +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--seed", type=int, default=20260927) diff --git a/scorecard/minimize.py b/scorecard/minimize.py index 6c3ac5d..ec76268 100644 --- a/scorecard/minimize.py +++ b/scorecard/minimize.py @@ -15,6 +15,7 @@ __all__ = ["minimize", "candidates"] +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _step_lists(steps, path=()): """(path, list) for the top-level list and every nested pipeline's list.""" yield path, steps @@ -23,6 +24,7 @@ def _step_lists(steps, path=()): yield from _step_lists(step["steps"], path + (i,)) +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _resolve(case, path): steps = case["steps"] for i in path: @@ -30,6 +32,7 @@ def _resolve(case, path): return steps +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def candidates(case): """Smaller variants of ``case``, most aggressive first.""" for path, steps in list(_step_lists(case["steps"])): @@ -69,6 +72,7 @@ def candidates(case): yield variant +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def minimize(case, still_fails, max_attempts=400): current = copy.deepcopy(case) attempts = 0 diff --git a/scorecard/modes.py b/scorecard/modes.py index a556506..5b22468 100644 --- a/scorecard/modes.py +++ b/scorecard/modes.py @@ -35,6 +35,7 @@ def describe_error(error): return f"{type(error).__name__}: {error}" +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _call(fn, data): vocab.sink.reset() try: @@ -55,6 +56,7 @@ def _run_twice(fn, case): return record +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def _export(pipeline, lowering, observe): try: return vocab.load_export(pipeline, lowering, observe, vocab.new_workdir()), None @@ -90,6 +92,7 @@ def fn(data): return record +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def run_fast(case, predicates, lowering): """Fast export. A case with Hooks must be *refused* (documented); the same pipeline without Hooks is then exported and compared on output, @@ -110,6 +113,7 @@ def run_fast(case, predicates, lowering): return record +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def run_observe(case, predicates): """Observable export. The pipeline carries the case's Hooks (so SOURCE_HOOKS is exercised); a SnapshotHook is passed at run time diff --git a/scorecard/snapshot_hook.py b/scorecard/snapshot_hook.py index ac03bfe..a78c64f 100644 --- a/scorecard/snapshot_hook.py +++ b/scorecard/snapshot_hook.py @@ -15,18 +15,13 @@ __all__ = ["SnapshotHook"] -def _as_dict(payload): - to_dict = getattr(payload, "to_dict", None) - return to_dict() if callable(to_dict) else payload - - class SnapshotHook: def __init__(self): self.events = [] # (event, step, snapshot, error or None) def _record(self, event, step, payload, error=None): err = None if error is None else f"{type(error).__name__}: {error}" - self.events.append((event, step, copy.deepcopy(_as_dict(payload)), err)) + self.events.append((event, step, copy.deepcopy(SnapshotHook._as_dict(payload)), err)) def before(self, step, payload): self._record("before", step, payload) @@ -41,3 +36,7 @@ def sequence(self): """[(event, step, snapshot, error)] for parity comparison.""" return list(self.events) + @staticmethod + def _as_dict(payload): + to_dict = getattr(payload, "to_dict", None) + return to_dict() if callable(to_dict) else payload diff --git a/scorecard/strict.py b/scorecard/strict.py index c7276d7..dc19e6f 100644 --- a/scorecard/strict.py +++ b/scorecard/strict.py @@ -8,6 +8,7 @@ __all__ = ["strict_equal", "first_difference"] +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def first_difference(a, b, path="$"): """None if ``a`` and ``b`` are strictly equal, else a description of the first place they differ.""" diff --git a/scorecard/vocab.py b/scorecard/vocab.py index 895d465..606e47b 100644 --- a/scorecard/vocab.py +++ b/scorecard/vocab.py @@ -1,7 +1,7 @@ """The conformance vocabulary, reused (not reimplemented) from the Python -conformance runner: ``runner.build`` turns a case's steps into a live -Pipeline, ``runner._predicate_source`` writes Valve predicates as real -``def``s, and ``runner._load_export`` exports and imports the package from +conformance runner: ``BuildPipelineFilter`` turns a case's steps into a live +Pipeline, ``WritePredicatesFilter`` writes Valve predicates as real +classes, and ``ExportFilter`` exports and imports the package from disk, as a customer would. The only thing done differently here: predicate ``def``s get index-based @@ -19,6 +19,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] RUNNER_DIR = ROOT / "conformance" / "runners" / "python" +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) if str(RUNNER_DIR) not in sys.path: @@ -26,11 +27,13 @@ import conformance_sink as sink # noqa: E402 import cup_ops as ops # noqa: E402 -import runner # noqa: E402 +from case_runner.build_pipeline_filter import BuildPipelineFilter # noqa: E402 +from case_runner.export_filter import ExportFilter # noqa: E402 +from case_runner.write_predicates_filter import WritePredicatesFilter # noqa: E402 from codeupipe import Payload, Pipeline # noqa: E402 -from codeupipe.lower.errors import UnsupportedFilterShape # noqa: E402 +from codeupipe.compile.errors import UnsupportedFilterShape # noqa: E402 -__all__ = ["sink", "ops", "runner", "Payload", "Pipeline", "UnsupportedFilterShape", +__all__ = ["sink", "ops", "Payload", "Pipeline", "UnsupportedFilterShape", "make_predicates", "build_pipeline", "load_export", "new_workdir", "walk_steps"] _COUNTER = itertools.count() @@ -40,6 +43,7 @@ def new_workdir(prefix="cup_scorecard_"): return pathlib.Path(tempfile.mkdtemp(prefix=prefix)) +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def walk_steps(steps): """Every step spec, depth first, nested pipelines included.""" for step in steps: @@ -48,11 +52,12 @@ def walk_steps(steps): yield from walk_steps(step["steps"]) +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def make_predicates(steps, workdir): """A namespace with ``when_`` for every Valve, each a real ``def`` in a real file (static lowering needs its source).""" valves = [s for s in walk_steps(steps) if s["kind"] == "valve"] - sources = [runner._predicate_source(str(i), s["when"]) for i, s in enumerate(valves)] + sources = [WritePredicatesFilter().source(str(i), s["when"]) for i, s in enumerate(valves)] name = f"scorecard_predicates_{next(_COUNTER)}" path = pathlib.Path(workdir) / f"{name}.py" path.write_text("\n\n".join(sources) or "\n") @@ -66,14 +71,15 @@ def make_predicates(steps, workdir): return namespace +# cup: allow-plain the differential test harness: it makes cases, runs them in each mode and compares the results def build_pipeline(case, predicates, hooks=()): - """The case as a live codeupipe Pipeline (runner.build), with ``hooks``.""" - pipeline = runner.build(case, Pipeline, predicates, {}) + """The case as a live codeupipe Pipeline (BuildPipelineFilter), with ``hooks``.""" + pipeline = BuildPipelineFilter().build(case["steps"], Pipeline, predicates, {}) for hook in hooks: pipeline.use_hook(hook) return pipeline def load_export(pipeline, lowering, observe, workdir): - """runner._load_export: export, write to disk, import package ``steps``.""" - return runner._load_export(pipeline, lowering, observe, pathlib.Path(workdir)) + """ExportFilter: export, write to disk, import package ``steps``.""" + return ExportFilter().load(pipeline, lowering, observe, pathlib.Path(workdir)) diff --git a/tests/lower/__init__.py b/tests/compile/__init__.py similarity index 100% rename from tests/lower/__init__.py rename to tests/compile/__init__.py diff --git a/tests/lower/fixtures/__init__.py b/tests/compile/fixtures/__init__.py similarity index 100% rename from tests/lower/fixtures/__init__.py rename to tests/compile/fixtures/__init__.py diff --git a/tests/lower/fixtures/cli_targets.py b/tests/compile/fixtures/cli_targets.py similarity index 73% rename from tests/lower/fixtures/cli_targets.py rename to tests/compile/fixtures/cli_targets.py index 51c33d4..8c91143 100644 --- a/tests/lower/fixtures/cli_targets.py +++ b/tests/compile/fixtures/cli_targets.py @@ -1,9 +1,9 @@ -"""Fixture: things `python -m codeupipe.lower module:name` can be pointed at.""" +"""Fixture: things `python -m codeupipe.compile module:name` can be pointed at.""" from codeupipe.core import Pipeline -from tests.lower.fixtures.instance_config import AtLeast, Discount, HoldsAnObject, SetFlag -from tests.lower.fixtures.plane_b import ExplodingTap +from tests.compile.fixtures.instance_config import AtLeast, Discount, HoldsAnObject, SetFlag +from tests.compile.fixtures.plane_b import ExplodingTap PRICING = Pipeline() PRICING.add_filter(Discount(0.1)) diff --git a/tests/lower/fixtures/export_module_globals.py b/tests/compile/fixtures/export_module_globals.py similarity index 100% rename from tests/lower/fixtures/export_module_globals.py rename to tests/compile/fixtures/export_module_globals.py diff --git a/tests/lower/fixtures/inline_shapes.py b/tests/compile/fixtures/inline_shapes.py similarity index 100% rename from tests/lower/fixtures/inline_shapes.py rename to tests/compile/fixtures/inline_shapes.py diff --git a/tests/lower/fixtures/instance_config.py b/tests/compile/fixtures/instance_config.py similarity index 100% rename from tests/lower/fixtures/instance_config.py rename to tests/compile/fixtures/instance_config.py diff --git a/tests/compile/fixtures/parameter_names.py b/tests/compile/fixtures/parameter_names.py new file mode 100644 index 0000000..6e06964 --- /dev/null +++ b/tests/compile/fixtures/parameter_names.py @@ -0,0 +1,51 @@ +"""Fixture: steps whose Payload parameter is not called ``payload``. The +export must follow the name the author chose.""" + +from codeupipe.core import Pipeline + + +class DiscountFilter: + def call(self, order): + return order.insert("price", round(order.get("price", 0) * 0.9, 2)) + + +class LabelFilter: + def call(self, p): + payload = "kept" # a local that happens to be called payload + same = p + same.insert("label", payload) + return same + + +class TotalFilter: + def call(self, data): + return data.insert("total", data.get("price", 0) * data.get("qty", 0)) + + +class MarkBulkFilter: + def call(self, order): + return order.insert("bulk", True) + + +class AtLeastPredicate: + def __init__(self, minimum): + self.minimum = minimum + + def test(self, order): + payload = self.minimum # a local that happens to be called payload + return order.get("qty", 0) >= payload + + +class AuditTap: + def observe(self, seen): + seen.get("price") + + +def build() -> Pipeline: + pipeline = Pipeline() + pipeline.add_filter(DiscountFilter()) + pipeline.add_filter(LabelFilter()) + pipeline.add_filter(TotalFilter()) + pipeline.add_valve(AtLeastPredicate(10), MarkBulkFilter()) + pipeline.add_tap(AuditTap()) + return pipeline diff --git a/tests/lower/fixtures/plane_b.py b/tests/compile/fixtures/plane_b.py similarity index 100% rename from tests/lower/fixtures/plane_b.py rename to tests/compile/fixtures/plane_b.py diff --git a/tests/lower/fixtures/typed_steps.py b/tests/compile/fixtures/typed_steps.py similarity index 100% rename from tests/lower/fixtures/typed_steps.py rename to tests/compile/fixtures/typed_steps.py diff --git a/tests/lower/test_cli.py b/tests/compile/test_cli.py similarity index 93% rename from tests/lower/test_cli.py rename to tests/compile/test_cli.py index d772f83..b80095c 100644 --- a/tests/lower/test_cli.py +++ b/tests/compile/test_cli.py @@ -1,4 +1,4 @@ -"""`python -m codeupipe.lower module:name --out DIR`: the export as a command.""" +"""`python -m codeupipe.compile module:name --out DIR`: the export as a command.""" import importlib import sys @@ -6,10 +6,10 @@ import pytest from codeupipe.core import Payload -from codeupipe.lower.__main__ import main -from tests.lower.fixtures import cli_targets +from codeupipe.compile.__main__ import main +from tests.compile.fixtures import cli_targets -TARGETS = "tests.lower.fixtures.cli_targets" +TARGETS = "tests.compile.fixtures.cli_targets" INPUTS = [{"price": 100, "qty": 12}, {"price": 80, "qty": 2}, {}] @@ -147,7 +147,7 @@ def test_runs_as_a_module(tmp_path): out = tmp_path / "out" done = subprocess.run( - [sys.executable, "-m", "codeupipe.lower", f"{TARGETS}:build_pricing", "--out", str(out)], + [sys.executable, "-m", "codeupipe.compile", f"{TARGETS}:build_pricing", "--out", str(out)], capture_output=True, text=True, ) diff --git a/tests/lower/test_emit_inline_module_filter.py b/tests/compile/test_emit_inline_module_filter.py similarity index 99% rename from tests/lower/test_emit_inline_module_filter.py rename to tests/compile/test_emit_inline_module_filter.py index 55f7f6c..20c1568 100644 --- a/tests/lower/test_emit_inline_module_filter.py +++ b/tests/compile/test_emit_inline_module_filter.py @@ -13,7 +13,7 @@ import pytest from codeupipe import Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 +from codeupipe.compile import build_export_pipeline_v2 from .fixtures import export_module_globals as g from .fixtures import inline_shapes as s diff --git a/tests/lower/test_emit_orchestrator_filter.py b/tests/compile/test_emit_orchestrator_filter.py similarity index 93% rename from tests/lower/test_emit_orchestrator_filter.py rename to tests/compile/test_emit_orchestrator_filter.py index f07353d..85b4aec 100644 --- a/tests/lower/test_emit_orchestrator_filter.py +++ b/tests/compile/test_emit_orchestrator_filter.py @@ -4,8 +4,8 @@ import textwrap from codeupipe import Payload -from codeupipe.lower.filters.emit_orchestrator_filter import EmitOrchestratorFilter -from codeupipe.lower.ir import PipelineIR, StepIR +from codeupipe.compile.filters.emit_orchestrator_filter import EmitOrchestratorFilter +from codeupipe.compile.ir import PipelineIR, StepIR def _plain_fn(name: str) -> ast.FunctionDef: diff --git a/tests/lower/test_emit_plain_files_filter.py b/tests/compile/test_emit_plain_files_filter.py similarity index 95% rename from tests/lower/test_emit_plain_files_filter.py rename to tests/compile/test_emit_plain_files_filter.py index 46df94e..98e895d 100644 --- a/tests/lower/test_emit_plain_files_filter.py +++ b/tests/compile/test_emit_plain_files_filter.py @@ -4,8 +4,8 @@ import textwrap from codeupipe import Payload -from codeupipe.lower.filters.emit_plain_files_filter import EmitPlainFilesFilter -from codeupipe.lower.ir import PipelineIR, StepIR +from codeupipe.compile.filters.emit_plain_files_filter import EmitPlainFilesFilter +from codeupipe.compile.ir import PipelineIR, StepIR def _plain_fn(name: str) -> ast.FunctionDef: diff --git a/tests/lower/test_errors.py b/tests/compile/test_errors.py similarity index 95% rename from tests/lower/test_errors.py rename to tests/compile/test_errors.py index 12444a9..12b3fd3 100644 --- a/tests/lower/test_errors.py +++ b/tests/compile/test_errors.py @@ -2,7 +2,7 @@ import pytest -from codeupipe.lower.errors import UnsupportedFilterShape +from codeupipe.compile.errors import UnsupportedFilterShape class TestUnsupportedFilterShape: diff --git a/tests/lower/test_export_instance_config.py b/tests/compile/test_export_instance_config.py similarity index 96% rename from tests/lower/test_export_instance_config.py rename to tests/compile/test_export_instance_config.py index 89606b6..97bebae 100644 --- a/tests/lower/test_export_instance_config.py +++ b/tests/compile/test_export_instance_config.py @@ -11,8 +11,8 @@ import pytest from codeupipe import Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 -from codeupipe.lower.errors import UnsupportedFilterShape +from codeupipe.compile import build_export_pipeline_v2 +from codeupipe.compile.errors import UnsupportedFilterShape from .fixtures import instance_config as c from .test_export_pipeline import _write_and_import diff --git a/tests/lower/test_export_module_globals.py b/tests/compile/test_export_module_globals.py similarity index 96% rename from tests/lower/test_export_module_globals.py rename to tests/compile/test_export_module_globals.py index 2ee8fce..93412b6 100644 --- a/tests/lower/test_export_module_globals.py +++ b/tests/compile/test_export_module_globals.py @@ -9,8 +9,8 @@ import pytest from codeupipe import Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 -from codeupipe.lower.errors import UnsupportedFilterShape +from codeupipe.compile import build_export_pipeline_v2 +from codeupipe.compile.errors import UnsupportedFilterShape from .fixtures import export_module_globals as g from .test_export_pipeline import _write_and_import diff --git a/tests/lower/test_export_observe.py b/tests/compile/test_export_observe.py similarity index 94% rename from tests/lower/test_export_observe.py rename to tests/compile/test_export_observe.py index 42f096c..50213c0 100644 --- a/tests/lower/test_export_observe.py +++ b/tests/compile/test_export_observe.py @@ -1,6 +1,6 @@ """Observable static export: ``observe=True``. -The fast export gives up State and Hooks (docs/static-export.md). With +The fast export gives up State and Hooks (docs/compile.md). With ``observe=True`` the exported ``run`` records the same State the dynamic ``run`` does and calls Hooks in the same order, so the two modes can be judged by the same conformance cases, State and Hooks included. @@ -12,18 +12,18 @@ import pytest from codeupipe import Hook, Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 +from codeupipe.compile import build_export_pipeline_v2 -ADD = f"{__name__}._Add" -BOOM = f"{__name__}._Boom" -FAILING_TAP = f"{__name__}._FailingTap" -TAP = f"{__name__}._Tap" -SUB = f"{__name__}.SubPipeline" +ADD = "_Add" +BOOM = "_Boom" +FAILING_TAP = "_FailingTap" +TAP = "_Tap" +SUB = "SubPipeline" class SubPipeline(Pipeline): - """A named pipeline type, so it can be nested (conformance/API.md, step identity).""" + """A named pipeline type, so it can be nested (conformance/API.md, step labels).""" class _Add: @@ -279,7 +279,7 @@ def test_no_framework_import(self, tmp_path): class TestFastExportIsUnchanged: def test_hooks_still_refused_without_observe(self, tmp_path): - from codeupipe.lower.errors import UnsupportedFilterShape + from codeupipe.compile.errors import UnsupportedFilterShape p = Pipeline() p.add_filter(_Add()) @@ -288,7 +288,7 @@ def test_hooks_still_refused_without_observe(self, tmp_path): _export(p, tmp_path, observe=False) def test_nested_pipeline_hooks_refused_even_when_observing(self, tmp_path): - from codeupipe.lower.errors import UnsupportedFilterShape + from codeupipe.compile.errors import UnsupportedFilterShape inner = SubPipeline() inner.add_filter(_Add()) @@ -301,7 +301,7 @@ def test_nested_pipeline_hooks_refused_even_when_observing(self, tmp_path): def test_empty_nested_pipeline_refused_when_observing(self, tmp_path): """Scorecard finding F1: it flattens to no steps, so the observable export could not report it in State and Hooks as the dynamic run does.""" - from codeupipe.lower.errors import UnsupportedFilterShape + from codeupipe.compile.errors import UnsupportedFilterShape outer = Pipeline() outer.add_filter(_Add()) @@ -317,8 +317,8 @@ def test_inline_layout_cannot_observe(self, tmp_path): _export(p, tmp_path, lowering="inline") -def test_identities_are_labels_not_code_names(tmp_path): - """Generated code names steps step_01_... itself; the identity (which is +def test_labels_are_not_code_names(tmp_path): + """Generated code names steps step_01_... itself; the label (which is not an identifier) is only ever a string in State and Hooks.""" p = Pipeline() p.add_filter(_Add()) diff --git a/tests/compile/test_export_parameter_name.py b/tests/compile/test_export_parameter_name.py new file mode 100644 index 0000000..77e6570 --- /dev/null +++ b/tests/compile/test_export_parameter_name.py @@ -0,0 +1,55 @@ +"""A step may call its Payload parameter what it likes (``order``, ``p``). +The export follows that name: the compiled code gives the same output as the +dynamic run, and a refusal names the parameter as the author wrote it.""" + +import subprocess +import sys + +import pytest + +from codeupipe.compile.__main__ import main +from codeupipe.core import Payload +from tests.compile.fixtures import parameter_names + +TARGET = "tests.compile.fixtures.parameter_names:build" +INPUTS = [{"price": 100.0, "qty": 12}, {"price": 19.99, "qty": 2}, {}] +MODES = [[], ["--lowering", "inline"], ["--observe"]] + + +@pytest.mark.parametrize("mode", MODES, ids=["steps", "inline", "observe"]) +def test_the_compiled_code_gives_the_same_output(tmp_path, mode): + out = tmp_path / "out" + assert main([TARGET, "--out", str(out)] + mode) == 0 + + for data in INPUTS: + script = "import sys; sys.path.insert(0, %r); import steps; print(steps.run(%r))" % (str(out / "src"), data) + done = subprocess.run([sys.executable, "-I", "-c", script], capture_output=True, text=True, cwd=tmp_path) + + assert done.returncode == 0, done.stderr + assert eval(done.stdout) == parameter_names.build().run(Payload(dict(data))).to_dict() + + +class _WritesAnAttribute: + def call(self, order): + order.price = 1 + return order + + +class _ReadsAnAttribute: + def call(self, order): + return order.insert("price", order.price) + + +@pytest.mark.parametrize("step", [_WritesAnAttribute, _ReadsAnAttribute]) +@pytest.mark.parametrize("lowering", ["steps", "inline"]) +def test_an_attribute_of_the_parameter_is_refused_by_its_own_name(step, lowering): + from codeupipe import Pipeline + from codeupipe.compile import build_export_pipeline_v2 + from codeupipe.compile.errors import UnsupportedFilterShape + + pipeline = Pipeline() + pipeline.add_filter(step()) + with pytest.raises(UnsupportedFilterShape) as refused: + build_export_pipeline_v2().run(Payload({"pipeline": pipeline, "pattern": "flat", "lowering": lowering})) + assert "order.price" in str(refused.value) + assert step.__name__ in str(refused.value) diff --git a/tests/lower/test_export_pipeline.py b/tests/compile/test_export_pipeline.py similarity index 95% rename from tests/lower/test_export_pipeline.py rename to tests/compile/test_export_pipeline.py index bad81ff..89daed6 100644 --- a/tests/lower/test_export_pipeline.py +++ b/tests/compile/test_export_pipeline.py @@ -11,11 +11,11 @@ import pytest from codeupipe import Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 +from codeupipe.compile import build_export_pipeline_v2 class SubPipeline(Pipeline): - """A named pipeline type, so it can be nested (conformance/API.md, step identity).""" + """A named pipeline type, so it can be nested (conformance/API.md, step labels).""" class _CleanText: @@ -141,10 +141,8 @@ def test_predicate_parameter_need_not_be_named_payload(self, tmp_path): assert package.run({"text": "a", "price": 10, "member": False})["price"] == 10 def test_lambda_predicate_is_refused_before_export(self): - """Anonymous predicates never reach export: the Valve refuses them.""" - from codeupipe import AnonymousStep - - with pytest.raises(AnonymousStep, match="lambda"): + """A lambda is not a Predicate: it has no test method.""" + with pytest.raises(TypeError, match=r"test\(payload\)"): self._pipeline(lambda payload: payload.get("member") is True) @@ -182,7 +180,7 @@ def test_base_exceptions_from_a_tap_still_propagate(self, tmp_path): class TestNestedPipelineExport: """A nested Pipeline lowers by flattening its steps into the parent (conformance: pipeline/nested-pipeline-is-a-step). Generated modules are - numbered in flattened order; identities stay labels.""" + numbered in flattened order; labels stay as they are.""" def test_nested_pipeline_exports_and_matches_dynamic(self, tmp_path): inner = SubPipeline() diff --git a/tests/lower/test_export_refusals.py b/tests/compile/test_export_refusals.py similarity index 95% rename from tests/lower/test_export_refusals.py rename to tests/compile/test_export_refusals.py index b1637f2..ff79b28 100644 --- a/tests/lower/test_export_refusals.py +++ b/tests/compile/test_export_refusals.py @@ -4,7 +4,7 @@ from the dynamic runtime: Hooks vanished, async steps became broken code, ``payload.trace_id`` became ``data.trace_id`` on a dict, and resilience wrappers died on an unrelated AttributeError. The contract is in -docs/static-export.md: anything that exports behaves the same, and anything +docs/compile.md: anything that exports behaves the same, and anything that cannot is refused at export time with the step named. """ @@ -13,8 +13,8 @@ import pytest from codeupipe import Hook, Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 -from codeupipe.lower.errors import UnsupportedFilterShape +from codeupipe.compile import build_export_pipeline_v2 +from codeupipe.compile.errors import UnsupportedFilterShape LOWERINGS = ["steps", "inline"] diff --git a/tests/lower/test_export_typed_steps.py b/tests/compile/test_export_typed_steps.py similarity index 91% rename from tests/lower/test_export_typed_steps.py rename to tests/compile/test_export_typed_steps.py index 77d2d7f..51644b8 100644 --- a/tests/lower/test_export_typed_steps.py +++ b/tests/compile/test_export_typed_steps.py @@ -6,10 +6,10 @@ import pytest from codeupipe.core import Payload -from codeupipe.lower.__main__ import main -from tests.lower.fixtures import typed_steps +from codeupipe.compile.__main__ import main +from tests.compile.fixtures import typed_steps -TARGET = "tests.lower.fixtures.typed_steps:build" +TARGET = "tests.compile.fixtures.typed_steps:build" INPUTS = [{"price": 2.5, "qty": 2}, {"price": 1.0, "qty": 12}] MODES = [[], ["--lowering", "inline"], ["--observe"]] @@ -56,7 +56,7 @@ def test_an_annotated_local_keeps_its_value_and_loses_its_type(tmp_path): def test_erasing_hints_never_leaves_an_empty_block(): import ast - from codeupipe.lower.filters.parse_filter_source_filter import _erase_annotations + from codeupipe.compile.filters.parse_filter_source_filter import _erase_annotations source = ( "def call(self, payload: 'P', *rest: int, flag: bool = False, **more: str) -> 'P':\n" diff --git a/tests/lower/test_ir.py b/tests/compile/test_ir.py similarity index 98% rename from tests/lower/test_ir.py rename to tests/compile/test_ir.py index bbcd34a..1b78b0c 100644 --- a/tests/lower/test_ir.py +++ b/tests/compile/test_ir.py @@ -5,7 +5,7 @@ import pytest -from codeupipe.lower.ir import ( +from codeupipe.compile.ir import ( HookIR, PipelineIR, PipelineMeta, diff --git a/tests/lower/test_load_pipeline_filter.py b/tests/compile/test_load_pipeline_filter.py similarity index 81% rename from tests/lower/test_load_pipeline_filter.py rename to tests/compile/test_load_pipeline_filter.py index 8ed41e0..a1b6a77 100644 --- a/tests/lower/test_load_pipeline_filter.py +++ b/tests/compile/test_load_pipeline_filter.py @@ -3,9 +3,9 @@ import pytest from codeupipe import Payload, Pipeline -from codeupipe.lower.errors import UnsupportedFilterShape -from codeupipe.lower.filters.load_pipeline_filter import LoadPipelineFilter -from codeupipe.lower.ir import PipelineIR +from codeupipe.compile.errors import UnsupportedFilterShape +from codeupipe.compile.filters.load_pipeline_filter import LoadPipelineFilter +from codeupipe.compile.ir import PipelineIR class _ExFilter: @@ -30,14 +30,14 @@ def test(self, payload): class TestLoadPipeline: - def test_extracts_step_identities_idents_and_kinds(self): + def test_extracts_step_labels_idents_and_kinds(self): p = Pipeline() p.add_filter(_ExFilter()) p.add_tap(_ExTap()) result = LoadPipelineFilter().call(Payload({"pipeline": p})) ir = result.get("pipeline_ir") assert isinstance(ir, PipelineIR) - assert [s.name for s in ir.steps] == [f"{__name__}._ExFilter", f"{__name__}._ExTap"] + assert [s.name for s in ir.steps] == ["_ExFilter", "_ExTap"] assert [s.ident for s in ir.steps] == ["step_01_ex_filter", "step_02_ex_tap"] assert [s.kind for s in ir.steps] == ["filter", "tap"] diff --git a/tests/lower/test_lower_control_flow_filter.py b/tests/compile/test_lower_control_flow_filter.py similarity index 87% rename from tests/lower/test_lower_control_flow_filter.py rename to tests/compile/test_lower_control_flow_filter.py index c789f64..7a151d9 100644 --- a/tests/lower/test_lower_control_flow_filter.py +++ b/tests/compile/test_lower_control_flow_filter.py @@ -4,8 +4,8 @@ import textwrap from codeupipe import Payload -from codeupipe.lower.filters.lower_control_flow_filter import LowerControlFlowFilter -from codeupipe.lower.ir import PipelineIR, StepIR +from codeupipe.compile.filters.lower_control_flow_filter import LowerControlFlowFilter +from codeupipe.compile.ir import PipelineIR, StepIR def _fn(src: str) -> ast.FunctionDef: @@ -30,7 +30,7 @@ def test_valve_predicate_is_dict_rewritten(self): assert "data.get" in pred_src def test_hooks_produce_warning_but_do_not_fail(self): - from codeupipe.lower.ir import HookIR + from codeupipe.compile.ir import HookIR ir = PipelineIR( steps=[], hooks=[HookIR(name="metrics")], diff --git a/tests/lower/test_parse_filter_source_filter.py b/tests/compile/test_parse_filter_source_filter.py similarity index 92% rename from tests/lower/test_parse_filter_source_filter.py rename to tests/compile/test_parse_filter_source_filter.py index fabd3b4..28ff50b 100644 --- a/tests/lower/test_parse_filter_source_filter.py +++ b/tests/compile/test_parse_filter_source_filter.py @@ -5,9 +5,9 @@ import pytest from codeupipe import Payload -from codeupipe.lower.errors import UnsupportedFilterShape -from codeupipe.lower.filters.parse_filter_source_filter import ParseFilterSourceFilter -from codeupipe.lower.ir import HookIR, PipelineIR, StepIR +from codeupipe.compile.errors import UnsupportedFilterShape +from codeupipe.compile.filters.parse_filter_source_filter import ParseFilterSourceFilter +from codeupipe.compile.ir import HookIR, PipelineIR, StepIR class _ExampleFilter: diff --git a/tests/lower/test_per_filter_imports.py b/tests/compile/test_per_filter_imports.py similarity index 97% rename from tests/lower/test_per_filter_imports.py rename to tests/compile/test_per_filter_imports.py index db8c619..b3827e3 100644 --- a/tests/lower/test_per_filter_imports.py +++ b/tests/compile/test_per_filter_imports.py @@ -5,7 +5,7 @@ import json from codeupipe import Payload, Pipeline -from codeupipe.lower import build_export_pipeline_v2 +from codeupipe.compile import build_export_pipeline_v2 class _UsesMath: diff --git a/tests/lower/test_rewrite_payload_ops_filter.py b/tests/compile/test_rewrite_payload_ops_filter.py similarity index 98% rename from tests/lower/test_rewrite_payload_ops_filter.py rename to tests/compile/test_rewrite_payload_ops_filter.py index 675fa05..d764737 100644 --- a/tests/lower/test_rewrite_payload_ops_filter.py +++ b/tests/compile/test_rewrite_payload_ops_filter.py @@ -6,9 +6,9 @@ import pytest from codeupipe import Payload -from codeupipe.lower.errors import UnsupportedFilterShape -from codeupipe.lower.filters.rewrite_payload_ops_filter import RewritePayloadOpsFilter -from codeupipe.lower.ir import PipelineIR, StepIR, PipelineMeta +from codeupipe.compile.errors import UnsupportedFilterShape +from codeupipe.compile.filters.rewrite_payload_ops_filter import RewritePayloadOpsFilter +from codeupipe.compile.ir import PipelineIR, StepIR, PipelineMeta def _fn(src: str) -> ast.FunctionDef: diff --git a/tests/lower/test_write_files_tap.py b/tests/compile/test_write_files_tap.py similarity index 95% rename from tests/lower/test_write_files_tap.py rename to tests/compile/test_write_files_tap.py index fa60ba2..31e999a 100644 --- a/tests/lower/test_write_files_tap.py +++ b/tests/compile/test_write_files_tap.py @@ -3,7 +3,7 @@ from pathlib import Path from codeupipe import Payload -from codeupipe.lower.taps.write_files_tap import WriteFilesTap +from codeupipe.compile.taps.write_files_tap import WriteFilesTap class TestWriteFilesTap: diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 6307807..f154969 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -5,11 +5,11 @@ import pytest from codeupipe.config import ConfigError, load_config -from codeupipe.lower.__main__ import main as compile_main +from codeupipe.compile.__main__ import main as compile_main from codeupipe_lint.anonymous_steps.__main__ import main as anonymous_main from codeupipe_lint.invariants.__main__ import main as invariants_main -TARGETS = "tests.lower.fixtures.cli_targets" +TARGETS = "tests.compile.fixtures.cli_targets" def write(directory, config, name="cup.config.json"): diff --git a/tests/conformance/test_bench_baseline.py b/tests/conformance/test_bench_baseline.py index 77b5633..85964f4 100644 --- a/tests/conformance/test_bench_baseline.py +++ b/tests/conformance/test_bench_baseline.py @@ -1,7 +1,9 @@ -"""The speed check: each benchmark number against its recorded baseline. +"""The comparison of benchmark numbers with a recorded baseline +(bench.py --check). No test and no CI job runs the comparison itself: a +shared machine is too noisy to judge speed. It is kept for a machine whose +resources are fixed, such as a constrained container. -A number is compared as a ratio to a reference workload timed beside it, so -the check does not depend on how fast or how busy the machine is. +A number is compared as a ratio to a reference workload timed beside it. """ import pathlib @@ -63,16 +65,31 @@ def test_a_new_number_with_no_baseline_fails(): def test_the_baseline_file_holds_a_ratio_for_every_number(): baseline = bench.load_baseline() assert baseline, "run: python conformance/bench.py --record" - for version, recorded in baseline.items(): - assert set(recorded["relative"]) == set(bench.BENCH_TARGETS), version + for key, recorded in baseline.items(): + runtime = "rust" if key == "rust" else "python" # the other keys are Python versions + want = {target for target in bench.BENCH_TARGETS if target.split(":")[0] == runtime} + assert set(recorded["relative"]) == want, key for target, row in recorded["relative"].items(): - assert row and all(ratio > 0 for ratio in row.values()), (version, target) + assert row and all(ratio > 0 for ratio in row.values()), (key, target) -@pytest.mark.perf -def test_nothing_is_slower_or_faster_than_its_baseline(): - if bench.python_key() not in bench.load_baseline(): - pytest.skip("no baseline for Python %s: python conformance/bench.py --record" % bench.python_key()) - done = subprocess.run([sys.executable, str(CONFORMANCE / "bench.py"), "--check"], - capture_output=True, text=True, timeout=600) - assert done.returncode == 0, done.stdout[-3000:] + done.stderr[-2000:] +def test_a_rust_number_is_kept_apart_from_the_python_version(): + # The speed of the Rust runtime does not depend on the Python that runs bench.py. + assert bench.baseline_key("python:run", python="3.12") == "3.12" + assert bench.baseline_key("rust:run", python="3.12") == "rust" + + +def test_the_check_compares_each_target_with_its_own_baseline(): + baseline = {"3.12": {"relative": {"python:run": {"m": 5.0}, "python:static-inline": {"m": 0.5}}}, + "3.9": {"relative": {"python:run": {"m": 9.0}}}, + "rust": {"relative": {"rust:run": {"m": 1.0}}}} + assert bench.baseline_for(baseline, ["python:run", "rust:run"], python="3.12") == { + "relative": {"python:run": {"m": 5.0}, "rust:run": {"m": 1.0}}} + + +def test_a_record_replaces_only_the_targets_that_were_measured(): + baseline = {"3.12": {"relative": {"python:run": {"m": 5.0}}, "us_per_run": {"python:run": {"m": 4.0}}}} + now = {"relative": {"rust:run": {"m": 1.0}}, "us_per_run": {"rust:run": {"m": 0.9}}} + bench.record_into(baseline, now, python="3.12") + assert baseline == {"3.12": {"relative": {"python:run": {"m": 5.0}}, "us_per_run": {"python:run": {"m": 4.0}}}, + "rust": {"relative": {"rust:run": {"m": 1.0}}, "us_per_run": {"rust:run": {"m": 0.9}}}} diff --git a/tests/conformance/test_case_runner.py b/tests/conformance/test_case_runner.py new file mode 100644 index 0000000..414fac4 --- /dev/null +++ b/tests/conformance/test_case_runner.py @@ -0,0 +1,110 @@ +"""The Python runner is two pipelines, so each of its steps is tested alone.""" + +import pathlib +import sys + +import pytest + +RUNNER = pathlib.Path(__file__).resolve().parents[2] / "conformance" / "runners" / "python" +sys.path.insert(0, str(RUNNER)) + +from codeupipe import Payload, Pipeline # noqa: E402 + +import conformance_sink as sink # noqa: E402 +from case_runner.attach_record_hook_filter import AttachRecordHookFilter # noqa: E402 +from case_runner.build_pipeline_filter import BuildPipelineFilter # noqa: E402 +from case_runner.name_steps_filter import NameStepsFilter # noqa: E402 +from case_runner.parse_target_filter import ParseTargetFilter # noqa: E402 +from case_runner.run_case_pipeline import RunCasePipeline # noqa: E402 +from case_runner.run_each_case_filter import RunEachCaseFilter # noqa: E402 +from case_runner.unsupported import Unsupported # noqa: E402 +from case_runner.write_predicates_filter import WritePredicatesFilter # noqa: E402 + +VALVE = {"kind": "valve", "name": "gate", "when": {"op": "truthy", "key": "on"}, "filter": {"op": "set", "key": "b", "value": 2}} +CASE = {"id": "t/one", "steps": [{"kind": "filter", "name": "first", "op": "set", "key": "a", "value": 1}, VALVE, + {"kind": "tap", "name": "audit", "op": "record", "key": "a"}], "input": {"on": True}} + + +@pytest.mark.parametrize("target, compiled, run_async, lowering, observe", [ + ("run", False, False, "", False), ("run_async", False, True, "", False), + ("static-steps", True, False, "steps", False), ("static-inline", True, False, "inline", False), + ("static-observe", True, False, "steps", True), +]) +def test_what_the_name_of_a_target_means(target, compiled, run_async, lowering, observe): + out = ParseTargetFilter().call(Payload({"target": target})) + assert (out.get("compiled"), out.get("run_async"), out.get("lowering"), out.get("observe")) == (compiled, run_async, lowering, observe) + + +def test_a_name_that_is_not_a_target_stops_the_runner(): + with pytest.raises(SystemExit, match="unknown target 'nope'"): + ParseTargetFilter().call(Payload({"target": "nope"})) + with pytest.raises(SystemExit, match="unknown bench target 'run_async'"): + ParseTargetFilter(("run",), "bench target").call(Payload({"target": "run_async"})) + + +def built(tmp_path, case=CASE, run_async=False): + payload = Payload({"case": case, "workdir": tmp_path, "module_name": f"t_predicates_{abs(hash(tmp_path))}", "run_async": run_async}) + return BuildPipelineFilter().call(WritePredicatesFilter().call(payload)) + + +def test_a_predicate_of_a_case_is_a_real_class_with_test(tmp_path): + predicates = WritePredicatesFilter().call(Payload({"case": CASE, "workdir": tmp_path, "module_name": "t_predicates_a"})).get("predicates") + assert predicates.when_gate.test(Payload({"on": 1})) is True + assert predicates.when_gate.test(Payload({})) is False + assert type(predicates.when_gate).__name__ == "When_gate" + + +def test_the_pipeline_of_a_case_and_the_name_of_each_label(tmp_path): + out = built(tmp_path) + assert isinstance(out.get("pipeline"), Pipeline) + assert out.get("label") == {"SetFilter": "first", "SetFilter#2": "gate", "RecordTap": "audit"} + + +@pytest.mark.parametrize("step, message", [ + ({"kind": "filter", "name": "a", "op": "nope"}, "unknown filter op 'nope'"), + ({"kind": "tap", "name": "a", "op": "nope"}, "unknown tap op 'nope'"), + ({"kind": "nope", "name": "a"}, "unknown step kind 'nope'"), +]) +def test_a_word_that_the_runner_does_not_know_is_unsupported(tmp_path, step, message): + with pytest.raises(Unsupported, match=message): + built(tmp_path, {"id": "t/x", "steps": [step], "input": {}}) + + +def test_a_hook_is_attached_for_a_case_that_asks_and_a_target_that_calls_hooks(tmp_path): + def hooks(case_hooks, compiled, observe): + payload = built(tmp_path, {**CASE, "hooks": case_hooks}).insert("compiled", compiled).insert("observe", observe) + return AttachRecordHookFilter().call(payload).get("hooks") + + assert hooks(False, False, False) is None + assert len(hooks(True, False, False)) == 1 # the dynamic run + assert hooks(True, True, False) is None # fast compiled code calls no Hooks + assert len(hooks(True, True, True)) == 1 # compiled with observe + + +def test_each_step_in_a_result_gets_the_name_of_the_case(): + result = {"output": {}, "state": {"executed": ["SetFilter", "Other"], "skipped": ["SetFilter#2"], "errors": []}, + "inner_states": [{"executed": ["SetFilter#2"], "skipped": [], "errors": ["SetFilter"]}], + "hooks": ["before:", "before:SetFilter", "error:SetFilter#2"]} + named = NameStepsFilter().call(Payload({"result": result, "label": {"SetFilter": "first", "SetFilter#2": "gate"}})).get("result") + assert named["state"] == {"executed": ["first", "Other"], "skipped": ["gate"], "errors": []} + assert named["inner_states"] == [{"executed": ["gate"], "skipped": [], "errors": ["first"]}] + assert named["hooks"] == ["before:", "before:first", "error:gate"] + + +@pytest.mark.parametrize("target", ["run", "run_async", "static-steps", "static-inline", "static-observe"]) +def test_one_case_on_each_target(tmp_path, target): + sink.reset() + out = RunCasePipeline().run(Payload({"case": CASE, "target": target, "workdir": tmp_path, "module_name": f"t_case_{target.replace('-', '_')}"})) + result = out.get("result") + assert result["output"] == {"on": True, "a": 1, "b": 2} + assert result["taps"] == [1] + assert result["input_after"] == {"on": True} + assert ("state" in result) == (target not in ("static-steps", "static-inline")) + assert result.get("state", {"executed": ["first", "gate", "audit"]})["executed"] == ["first", "gate", "audit"] + + +def test_a_case_that_a_target_cannot_express_is_reported_and_the_next_case_still_runs(): + cases = [{"id": "t/bad", "steps": [{"kind": "filter", "name": "a", "op": "nope"}], "input": {}}, CASE] + results = RunEachCaseFilter(RunCasePipeline()).call(Payload({"cases": cases, "target": "run"})).get("results") + assert results[0] == {"id": "t/bad", "unsupported": "unknown filter op 'nope'"} + assert results[1]["id"] == "t/one" and results[1]["output"] == {"on": True, "a": 1, "b": 2} diff --git a/tests/conformance/test_judge_pipeline.py b/tests/conformance/test_judge_pipeline.py new file mode 100644 index 0000000..00fa78b --- /dev/null +++ b/tests/conformance/test_judge_pipeline.py @@ -0,0 +1,119 @@ +"""The judge is a pipeline, so each of its steps is tested alone, with no runner and no process.""" + +import json +import pathlib +import sys + +import pytest + +CONFORMANCE = pathlib.Path(__file__).resolve().parents[2] / "conformance" +sys.path.insert(0, str(CONFORMANCE)) + +from codeupipe import Payload # noqa: E402 + +from judging.default_targets_filter import DefaultTargetsFilter # noqa: E402 +from judging.judge_pipeline import JudgePipeline # noqa: E402 +from judging.judge_results_filter import JudgeResultsFilter # noqa: E402 +from judging.load_cases_filter import LoadCasesFilter # noqa: E402 + +CASE = {"id": "a/one", "steps": [], "input": {"n": 1}, + "expect": {"output": {"n": 2}, "state": {"executed": ["s"], "skipped": [], "errors": []}, "hooks": ["before:"]}} +GOOD = {"id": "a/one", "output": {"n": 2.0}, "state": {"executed": ["s"], "skipped": [], "errors": []}, + "hooks": ["before:"], "taps": [], "input_after": {"n": 1}} + + +def without(result, *keys): + return {key: value for key, value in result.items() if key not in keys} + + +@pytest.mark.parametrize("result, status, problem", [ + (GOOD, "pass", None), + (None, "FAIL", "no result from runner"), + ({"id": "a/one", "unsupported": "no nested pipeline"}, "unsupported", "no nested pipeline"), + (without(GOOD, "state", "hooks"), "partial", "target cannot observe state, hooks"), + ({**GOOD, "output": {"n": 3}}, "FAIL", "output {'n': 3} != {'n': 2}"), + ({**GOOD, "output": {"n": True}}, "FAIL", "output {'n': True} != {'n': 2}"), + ({**without(GOOD, "output"), "error": "ValueError: no"}, "FAIL", "expected output, got error 'ValueError: no'"), + ({**GOOD, "input_after": {"n": 9}}, "FAIL", "input was mutated: {'n': 9}"), + ({**GOOD, "state": {"executed": [], "skipped": [], "errors": []}}, "FAIL", "state.executed [] != ['s']"), + ({**GOOD, "hooks": []}, "FAIL", "hooks [] != ['before:']"), +]) +def test_the_verdict_for_one_result(result, status, problem): + got_status, problems = JudgeResultsFilter().judge(CASE, result) + assert got_status == status + assert problems == ([] if problem is None else [problem]) + + +def test_an_error_is_expected_by_a_part_of_its_message(): + case = {"id": "a/two", "steps": [], "input": {}, "expect": {"error": {"message_contains": "boom"}}} + judge = JudgeResultsFilter().judge + assert judge(case, {"error": "ValueError: boom!", "input_after": {}})[0] == "pass" + assert judge(case, {"error": "ValueError: other", "input_after": {}})[0] == "FAIL" + assert judge(case, {"output": {}, "input_after": {}})[0] == "FAIL" + + +def test_a_python_target_always_runs_and_a_typescript_target_needs_node_and_its_files(tmp_path): + there = tmp_path / "built.js" + there.write_text("") + targets = {"python:run": {}, "typescript:run": {}, "typescript:static": {}, "typescript:other": {}} + needs = {"typescript:static": [there], "typescript:other": [tmp_path / "missing.js"]} + assert DefaultTargetsFilter(lambda: True).choose(targets, needs) == ["python:run", "typescript:run", "typescript:static"] + assert DefaultTargetsFilter(lambda: False).choose(targets, needs) == ["python:run"] + + +def test_a_rust_target_needs_cargo(): + targets = {"python:run": {}, "rust:run": {}} + assert DefaultTargetsFilter(lambda: False, lambda: True).choose(targets, {}) == ["python:run", "rust:run"] + assert DefaultTargetsFilter(lambda: False, lambda: False).choose(targets, {}) == ["python:run"] + + +def test_two_cases_with_one_id_stop_the_run(tmp_path): + (tmp_path / "a.json").write_text(json.dumps([{"id": "x"}, {"id": "x"}])) + with pytest.raises(SystemExit, match="duplicate case ids"): + LoadCasesFilter().load(tmp_path) + + +class FakeLauncher: + """A launcher that starts no process: it gives the results that the test made.""" + + def __init__(self, by_command): + self.by_command = by_command + + def run(self, spec, cases): + return self.by_command[spec["cmd"]] + + +def test_the_whole_judge_with_no_process(tmp_path): + cases_dir, results_dir = tmp_path / "cases", tmp_path / "results" + cases_dir.mkdir() + (cases_dir / "a.json").write_text(json.dumps([CASE])) + launcher = FakeLauncher({ + "good": ({"a/one": GOOD}, None), + "wrong": ({"a/one": {**GOOD, "output": {"n": 3}}}, None), + "broken": (None, "runner exited 3: no"), + }) + targets = {"t:good": {"cmd": "good"}, "t:wrong": {"cmd": "wrong"}, "t:broken": {"cmd": "broken"}} + pipeline = JudgePipeline(launcher, lambda: False) + + def run(names): + return pipeline.run(Payload({"cases_dir": cases_dir, "results_dir": results_dir, "targets": targets, + "needs": {}, "names": names})) + + out = run(["t:good"]) + assert out.get("exit_code") == 0 + assert out.get("matrix") == {"a/one": {"t:good": "pass"}} + assert json.loads((results_dir / "latest.json").read_text()) == out.get("report") + assert (results_dir / "latest.md").read_text() == out.get("markdown") + + out = run(["t:good", "t:wrong", "t:broken"]) + assert out.get("exit_code") == 1 + assert out.get("matrix") == {"a/one": {"t:good": "pass", "t:wrong": "FAIL", "t:broken": "—"}} + assert out.get("report")["targets"]["t:wrong"]["failures"] == {"a/one": ["output {'n': 3} != {'n': 2}"]} + assert out.get("report")["targets"]["t:broken"] == {"error": "runner exited 3: no"} + assert "- **t:broken**: not run: runner exited 3: no" in out.get("markdown") + + # With no target named, the judge takes each target that the machine can run. + assert set(run([]).get("matrix")["a/one"]) == {"t:good", "t:wrong", "t:broken"} + assert pipeline.state.executed[1] == "DefaultTargetsFilter" + run(["t:good"]) + assert pipeline.state.skipped == ["DefaultTargetsFilter"] diff --git a/tests/conformance/test_python_targets.py b/tests/conformance/test_python_targets.py index ba82b20..2aaa2ac 100644 --- a/tests/conformance/test_python_targets.py +++ b/tests/conformance/test_python_targets.py @@ -16,6 +16,10 @@ PYTHON_TARGETS = ["python:run", "python:run_async", "python:static-steps", "python:static-inline", "python:static-observe"] +# `state` on the Pipeline is a part of the Pipeline object. A compiled form is a function and has no +# such object, so it cannot show what this one case asks for. +NEEDS_A_PIPELINE_OBJECT = ["state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last"] + @pytest.fixture(scope="module") def report(tmp_path_factory): @@ -32,6 +36,8 @@ def test_target_passes_every_case_it_can_express(report, target): def test_observable_export_is_judged_on_state_and_hooks_too(report): - """No partial passes: static-observe exposes everything the dynamic run does.""" + """static-observe exposes the State and the Hooks that the dynamic run does, so it passes + each case outright, but for the one that asks for the State on the Pipeline object.""" summary = report["targets"]["python:static-observe"] - assert summary["pass"] == report["cases"], summary + assert summary["unsupported"] == 0, summary + assert summary["pass"] == report["cases"] - len(NEEDS_A_PIPELINE_OBJECT), summary diff --git a/tests/conformance/test_rust_target.py b/tests/conformance/test_rust_target.py new file mode 100644 index 0000000..5953cf7 --- /dev/null +++ b/tests/conformance/test_rust_target.py @@ -0,0 +1,32 @@ +"""Rust must pass the same conformance cases as Python. + +The test needs cargo. On a machine with no cargo it is skipped. +""" + +import json +import pathlib +import shutil +import sys + +import pytest + +CONFORMANCE = pathlib.Path(__file__).resolve().parents[2] / "conformance" +sys.path.insert(0, str(CONFORMANCE)) + +import run as conformance_run # noqa: E402 + + + +@pytest.mark.skipif(shutil.which("cargo") is None, reason="needs cargo") +def test_rust_passes_every_case(): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target("rust:run", conformance_run.TARGETS["rust:run"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + not_passed = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] != "pass"} + assert not_passed == {}, json.dumps(not_passed, indent=2) + + +def test_cargo_is_not_needed_for_the_default_targets(monkeypatch): + monkeypatch.setattr(conformance_run, "_has_cargo", lambda: False) + assert [name for name in conformance_run.default_targets() if name.startswith("rust:")] == [] diff --git a/tests/conformance/test_typescript_target.py b/tests/conformance/test_typescript_target.py new file mode 100644 index 0000000..fa1f590 --- /dev/null +++ b/tests/conformance/test_typescript_target.py @@ -0,0 +1,55 @@ +"""TypeScript must pass the same conformance cases as Python. + +The test needs a Node that runs a .ts file as it is (22.18 or later). On a +machine with no such Node it is skipped. +""" + +import json +import pathlib +import sys + +import pytest + +CONFORMANCE = pathlib.Path(__file__).resolve().parents[2] / "conformance" +sys.path.insert(0, str(CONFORMANCE)) + +import run as conformance_run # noqa: E402 + +pytestmark = pytest.mark.skipif(not conformance_run._node_runs_typescript(), + reason="needs Node 22.18 or later") + + +def test_typescript_passes_every_case(tmp_path): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target( + "typescript:run", conformance_run.TARGETS["typescript:run"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + not_passed = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] != "pass"} + assert not_passed == {}, json.dumps(not_passed, indent=2) + + +@pytest.mark.skipif("typescript:static" not in conformance_run.default_targets(), + reason="needs the compile package to be installed") +def test_the_compiled_pipeline_passes_every_case_on_what_it_can_show(): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target( + "typescript:static", conformance_run.TARGETS["typescript:static"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + wrong = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] not in ("pass", "partial")} + assert wrong == {}, json.dumps(wrong, indent=2) + assert sum(1 for verdict in verdicts.values() if verdict[0] == "pass") == 11 + + +def test_a_target_that_cannot_run_is_a_failure(monkeypatch, tmp_path): + monkeypatch.setattr(conformance_run, "RESULTS", tmp_path) + monkeypatch.setitem(conformance_run.TARGETS, "broken:run", + {"cmd": [sys.executable, "-c", "raise SystemExit(3)"], "env": {}, "cwd": tmp_path}) + assert conformance_run.main(["broken:run"]) == 1 + + +def test_node_is_not_needed_for_the_default_targets(monkeypatch): + monkeypatch.setattr(conformance_run, "_node_runs_typescript", lambda: False) + assert [name for name in conformance_run.default_targets() if name.startswith("typescript:")] == [] + assert len([name for name in conformance_run.default_targets() if name.startswith("python:")]) == 5 diff --git a/tests/core/test_add_valve.py b/tests/core/test_add_valve.py index 3a10008..f274dae 100644 --- a/tests/core/test_add_valve.py +++ b/tests/core/test_add_valve.py @@ -42,7 +42,7 @@ def test_the_filter_runs_when_the_predicate_is_true(): pipeline = Pipeline() pipeline.add_valve(IsOn(), SetA()) assert pipeline.run(Payload({"on": True})).to_dict() == {"on": True, "a": 1} - assert pipeline.state.executed == [f"{HERE}.SetA"] + assert pipeline.state.executed == ["SetA"] assert pipeline.state.skipped == [] @@ -51,20 +51,14 @@ def test_the_payload_passes_through_when_the_predicate_is_false(): pipeline.add_valve(IsOn(), SetA()) assert pipeline.run(Payload({"on": False})).to_dict() == {"on": False} assert pipeline.state.executed == [] - assert pipeline.state.skipped == [f"{HERE}.SetA"] + assert pipeline.state.skipped == ["SetA"] def test_a_valve_is_identified_by_its_filter(): pipeline = Pipeline() pipeline.add_valve(IsOn(), SetA()) pipeline.add_valve(AtLeast("n", 2), SetA()) - assert pipeline.identities == [f"{HERE}.SetA", f"{HERE}.SetA#2"] - - -def test_the_step_site_is_the_add_valve_line(): - pipeline = Pipeline() - pipeline.add_valve(IsOn(), SetA()) - assert "test_add_valve.py:" in pipeline.step_sites[f"{HERE}.SetA"] + assert pipeline.labels == ["SetA", "SetA#2"] def test_a_predicate_keeps_its_settings(): @@ -91,7 +85,7 @@ def test_an_async_filter_behind_a_valve_needs_run_async(): def test_a_function_is_not_a_predicate(): def is_on(payload): return True - with pytest.raises(codeupipe.AnonymousStep): + with pytest.raises(TypeError, match=r"test\(payload\)"): Pipeline().add_valve(is_on, SetA()) # cup: allow-anonymous (exercises the refusal) diff --git a/tests/core/test_api_contract.py b/tests/core/test_api_contract.py index 35c7d9a..95e17ce 100644 --- a/tests/core/test_api_contract.py +++ b/tests/core/test_api_contract.py @@ -21,8 +21,8 @@ async def call(self, payload): return payload.insert("n", payload.get("n", 0) + 1) -INC = f"{__name__}.Inc" -BOOM = f"{__name__}.Boom" +INC = "Inc" +BOOM = "Boom" class _SetsX: @@ -75,11 +75,11 @@ def test_nested_pipeline_runs_synchronously_as_one_step(self): outer.add_filter(Inc()) outer.add_pipeline(IncTwice()) assert outer.run(Payload({})).get("n") == 2 - assert outer.state.executed == [INC, f"{__name__}.IncTwice"] + assert outer.state.executed == [INC, "IncTwice"] class TestStepsHaveNoNames: - """Identity is derived from the filter (tests/core/test_step_identity.py).""" + """The label is derived from the step (tests/core/test_step_labels.py).""" @pytest.mark.parametrize("add", ["add_filter", "add_tap"]) def test_a_name_argument_is_rejected(self, add): @@ -93,8 +93,8 @@ def call(self, payload): raise RuntimeError('boom') -class TestHooksReceiveStepIdentities: - def test_before_after_and_on_error_get_identities(self): +class TestHooksReceiveStepLabels: + def test_before_after_and_on_error_get_labels(self): events = [] class Record(Hook): diff --git a/tests/core/test_custom_payload.py b/tests/core/test_custom_payload.py new file mode 100644 index 0000000..01bccdb --- /dev/null +++ b/tests/core/test_custom_payload.py @@ -0,0 +1,151 @@ +"""A pipeline over a type of your own. + +Payload is the default: a map that any developer can read. A type of your +own can take its place. The Pipeline asks one thing of it, ``copy()``, for +the copy that a run works on. Lineage is kept only by a type that has +``_stamp(label)``; for any other type, turning lineage on does nothing. +""" + +import asyncio +import dataclasses +from typing import Optional + +import pytest + +from codeupipe import Hook, Pipeline + + +@dataclasses.dataclass +class Order: + price: float + qty: int + bulk: bool = False + + def copy(self): + return dataclasses.replace(self) + + +@dataclasses.dataclass +class TracedOrder(Order): + steps: list = dataclasses.field(default_factory=list) + + def copy(self): + return dataclasses.replace(self, steps=list(self.steps)) + + def _stamp(self, label): + self.steps.append(label) + + +class DiscountFilter: + def call(self, order): + order.price = round(order.price * 0.9, 2) + return order + + +class MarkBulkFilter: + def call(self, order): + order.bulk = True + return order + + +class RefuseFreeFilter: + def call(self, order): + if order.price <= 0: + raise ValueError("an order must cost something") + return order + + +class AtLeastPredicate: + def test(self, order): + return order.qty >= 10 + + +class SeeTap: + def __init__(self): + self.seen = [] + + def observe(self, order): + self.seen.append(order.price) + + +class RecordHook(Hook): + def __init__(self): + super().__init__() + self.events = [] + + def before(self, step: Optional[str], payload) -> None: + self.events.append(f"before:{step or ''}") + + def on_error(self, step: Optional[str], error: Exception, payload) -> None: + self.events.append(f"error:{step or ''}:{payload.price}") + + +class PricingPipeline(Pipeline): + def __init__(self): + super().__init__() + self.add_filter(DiscountFilter()) + self.add_valve(AtLeastPredicate(), MarkBulkFilter()) + + +def _pricing(): + return PricingPipeline() + + +def test_every_primitive_takes_the_type(): + tap, hook = SeeTap(), RecordHook() + pipeline = _pricing() + pipeline.add_tap(tap) + pipeline.use_hook(hook) + given = Order(100.0, 12) + out = pipeline.run(given) + assert out == Order(90.0, 12, True) + assert given == Order(100.0, 12) # the caller's value is not changed + assert tap.seen == [90.0] + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter", "SeeTap"] + assert hook.events == ["before:", "before:DiscountFilter", "before:MarkBulkFilter"] + pipeline.run(Order(100.0, 2)) + assert pipeline.state.skipped == ["MarkBulkFilter"] + + +def test_the_fast_path_and_run_async_take_the_type(): + assert _pricing().run(Order(100.0, 12)) == Order(90.0, 12, True) + assert asyncio.run(_pricing().run_async(Order(100.0, 12))) == Order(90.0, 12, True) + + +def test_a_nested_pipeline_takes_the_type(): + pipeline = Pipeline() + pipeline.add_pipeline(PricingPipeline()) + pipeline.add_filter(RefuseFreeFilter()) + assert pipeline.run(Order(50.0, 10)) == Order(45.0, 10, True) + + +def test_a_failure_is_the_same_as_with_a_payload(): + hook = RecordHook() + pipeline = Pipeline() + pipeline.add_filter(DiscountFilter()) + pipeline.add_filter(RefuseFreeFilter()) + pipeline.use_hook(hook) + with pytest.raises(ValueError, match="must cost something"): + pipeline.run(Order(0.0, 1)) + assert pipeline.state.executed == ["DiscountFilter"] + assert hook.events[-2:] == ["error:RefuseFreeFilter:0.0", "error::0.0"] + + +@pytest.mark.parametrize("run", ["sync", "async"]) +def test_lineage_does_nothing_for_a_type_without_stamp(run): + pipeline = _pricing() + pipeline.observe(lineage=True) + given = Order(100.0, 12) + out = pipeline.run(given) if run == "sync" else asyncio.run(pipeline.run_async(given)) + assert out == Order(90.0, 12, True) + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter"] + + +@pytest.mark.parametrize("run", ["sync", "async"]) +def test_lineage_is_kept_by_a_type_with_stamp(run): + pipeline = _pricing() + pipeline.observe(lineage=True) + given = TracedOrder(100.0, 12) + out = pipeline.run(given) if run == "sync" else asyncio.run(pipeline.run_async(given)) + assert out.steps == ["DiscountFilter", "MarkBulkFilter"] + assert given.steps == [] diff --git a/tests/core/test_execution_modes.py b/tests/core/test_execution_modes.py index 2f1ff0a..c0d5e74 100644 --- a/tests/core/test_execution_modes.py +++ b/tests/core/test_execution_modes.py @@ -20,7 +20,7 @@ def test(self, payload): class SubPipeline(Pipeline): - """A named pipeline type, so it can be nested (conformance/API.md, step identity).""" + """A named pipeline type, so it can be nested (conformance/API.md, step labels).""" # ── Helpers ────────────────────────────────────────────────── @@ -123,7 +123,7 @@ def test_run_sync_updates_state(self): pipe = Pipeline() pipe.add_filter(AddTenFilter()) result = pipe.run(Payload({"value": 0})) - assert f"{__name__}.AddTenFilter" in pipe.state.executed + assert "AddTenFilter" in pipe.state.executed # ═══════════════════════════════════════════════════════════════ @@ -184,7 +184,7 @@ async def test_nested_pipeline_state_tracked(self): outer.add_pipeline(inner) await outer.run_async(Payload({"value": 0})) - assert f"{__name__}.SubPipeline" in outer.state.executed + assert "SubPipeline" in outer.state.executed @pytest.mark.asyncio async def test_nested_pipeline_error_propagates(self): diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index c5e800e..0f1df60 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -22,11 +22,11 @@ def __init__(self): self.log = [] async def before(self, step: Optional[str], payload: Payload) -> None: - # hooks receive the step's identity (API contract), None for the pipeline + # hooks receive the step's label (API contract), None for the pipeline self.log.append(f"before_{step if step is not None else 'pipeline_start'}") async def after(self, step: Optional[str], payload: Payload) -> None: - # hooks receive the step's identity (API contract), None for the pipeline + # hooks receive the step's label (API contract), None for the pipeline self.log.append(f"after_{step if step is not None else 'pipeline_end'}") async def on_error(self, step: Optional[str], error: Exception, payload: Payload) -> None: @@ -117,7 +117,7 @@ def test_pipeline_state_tracking(self): async def run_test(): await pipeline.run_async(Payload()) - assert pipeline.state.executed == [f"{__name__}.TestFilter2", f"{__name__}.TestFilter2#2"] + assert pipeline.state.executed == ["TestFilter2", "TestFilter2#2"] asyncio.run(run_test()) @@ -222,8 +222,8 @@ def test_valve_blocks_when_predicate_false(self): async def run_test(): result = await pipeline.run_async(Payload()) - assert pipeline.state.executed == [f"{__name__}.MarkFirst"] - assert pipeline.state.skipped == [f"{__name__}.MarkGated"] + assert pipeline.state.executed == ["MarkFirst"] + assert pipeline.state.skipped == ["MarkGated"] assert result.get("first_ran") is True assert result.get("gated_ran") is None @@ -289,7 +289,7 @@ def test_tap_tracked_in_state(self): async def run_test(): await pipeline.run_async(Payload()) - assert f"{__name__}.NoopTap" in pipeline.state.executed + assert "NoopTap" in pipeline.state.executed asyncio.run(run_test()) @@ -326,9 +326,9 @@ async def run_test(): await pipeline.run_async(Payload()) assert "before_pipeline_start" in hook.log - # hooks receive the step's identity (API contract) - assert f"before_{__name__}.NamedFilter3" in hook.log - assert f"after_{__name__}.NamedFilter3" in hook.log + # hooks receive the step's label (API contract) + assert "before_NamedFilter3" in hook.log + assert "after_NamedFilter3" in hook.log assert "after_pipeline_end" in hook.log asyncio.run(run_test()) @@ -436,7 +436,7 @@ def test_run_sync_records_a_closed_valve_as_skipped(self): result = p.run(Payload({})) assert result.get("marked") is None - assert p.state.skipped == [f"{__name__}.TestRunSyncWithValves._Mark"] + assert p.state.skipped == ["_Mark"] assert p.state.executed == [] @pytest.mark.unit @@ -450,7 +450,7 @@ def test_a_valve_over_an_async_filter_needs_run_async(self): p.run(Payload({})) result = asyncio.run(p.run_async(Payload({}))) assert result.get("marked") is True - assert p.state.executed == [f"{__name__}.TestRunSyncWithValves._AsyncMark"] + assert p.state.executed == ["_AsyncMark"] class _Predicate5: @@ -503,11 +503,11 @@ def _pipeline(self, tap, hook=None): def _assert_contained(self, p, result): assert result.get("score") == 2 - tap = next(i for i in p.identities if i.endswith(("._Boom", "._AsyncBoom"))) + tap = next(i for i in p.labels if i in ("_Boom", "_AsyncBoom")) assert p.state.errors[-1][0] == tap assert isinstance(p.state.errors[-1][1], ConnectionError) assert tap in p.state.executed - assert f"{__name__}.TestTapFailureIsolation._Score#2" in p.state.executed + assert "_Score#2" in p.state.executed @pytest.mark.unit @pytest.mark.core diff --git a/tests/core/test_state_per_run.py b/tests/core/test_state_per_run.py new file mode 100644 index 0000000..dcf9b5f --- /dev/null +++ b/tests/core/test_state_per_run.py @@ -0,0 +1,91 @@ +"""A State belongs to one run (conformance/API.md, "State"). + +Two runs of one Pipeline that are in progress at the same time each have +their own State. The conformance cases check a run that starts inside +another run. These tests check runs that really overlap, with asyncio. +""" + +import asyncio + +from codeupipe import Payload, Pipeline, State + + +class SlowFilter: + async def call(self, payload): + await asyncio.sleep(payload.get("wait")) + return payload.insert("done", payload.get("name")) + + +class FailForBFilter: + def call(self, payload): + if payload.get("name") == "B": + raise ValueError("B fails") + return payload + + +class IsAPredicate: + def test(self, payload): + return payload.get("name") == "A" + + +class FailTap: + def observe(self, payload): + if payload.get("name") == "B": + raise RuntimeError("the sink is down") + + +def _overlap(pipeline, give_state): + """Run A (starts first, ends last) and B (starts second, ends first) at the same time.""" + async def one(name, wait): + state = State() if give_state else None + try: + await pipeline.run_async(Payload({"name": name, "wait": wait}), state) + except ValueError: + pass + return state + + async def both(): + return await asyncio.gather(one("A", 0.05), one("B", 0.01)) + + return asyncio.run(both()) + + +def test_each_of_two_runs_at_the_same_time_writes_into_the_state_that_its_caller_gave(): + pipeline = Pipeline() + pipeline.add_filter(SlowFilter()) + pipeline.add_tap(FailTap()) + pipeline.add_filter(FailForBFilter()) + state_a, state_b = _overlap(pipeline, give_state=True) + assert state_a.executed == ["SlowFilter", "FailTap", "FailForBFilter"] + assert state_a.errors == [] + assert state_b.executed == ["SlowFilter", "FailTap"] + assert [label for label, _error in state_b.errors] == ["FailTap"] + + +def test_with_no_state_from_the_caller_the_steps_of_two_runs_do_not_mix(): + pipeline = Pipeline() + pipeline.add_filter(SlowFilter()) + pipeline.add_filter(FailForBFilter()) + _overlap(pipeline, give_state=False) + # B started last, so the Pipeline has the State of B: the one step that B did, and no step of A. + assert pipeline.state.executed == ["SlowFilter"] + + +def test_a_valve_with_an_async_filter_is_skipped_or_run_for_its_own_run(): + pipeline = Pipeline() + pipeline.add_valve(IsAPredicate(), SlowFilter()) + state_a, state_b = _overlap(pipeline, give_state=True) + assert (state_a.executed, state_a.skipped) == (["SlowFilter"], []) + assert (state_b.executed, state_b.skipped) == ([], ["SlowFilter"]) + + +def test_the_pipeline_has_the_state_that_the_caller_gave(): + pipeline = Pipeline() + pipeline.add_filter(FailForBFilter()) + state = State() + pipeline.run(Payload({"name": "A"}), state) + assert pipeline.state is state + assert state.executed == ["FailForBFilter"] + pipeline.run(Payload({"name": "A"})) + assert pipeline.state is not state + assert state.executed == ["FailForBFilter"] # a later run does not write into it diff --git a/tests/core/test_step_identity.py b/tests/core/test_step_identity.py deleted file mode 100644 index a6c2d9c..0000000 --- a/tests/core/test_step_identity.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Step identity: a pointer to the filter, derived from the code (conformance/API.md). - -No step names. A step is identified by its filter's fully qualified symbol, -``#n`` when the same one is used again in a pipeline; the add site -(file:line) is recorded as extra detail. Anonymous steps are refused. -""" - -import pytest - -from codeupipe import Hook, Payload, Pipeline -from codeupipe.core.valve import Valve -from codeupipe.core import AnonymousStep - -MODULE = __name__ - - -class Add: - def call(self, payload): - return payload.insert("n", payload.get("n", 0) + 1) - - -class Boom: - def call(self, payload): - raise RuntimeError("boom") - - -class Audit: - def observe(self, payload): - pass - - -class UserOnboarding(Pipeline): - """A named pipeline type: nestable, identified like a filter.""" - - def __init__(self): - super().__init__() - self.add_filter(Add()) - - -class IsOn: - """A predicate object: test(payload) -> bool, identified by its class.""" - - def test(self, payload): - return bool(payload.get("on")) - - -class AtLeast: - """A configurable predicate: its setting lives on the instance.""" - - def __init__(self, n): - self.n = n - - def test(self, payload): - return payload.get("n", 0) >= self.n - - -def is_on(payload): - return bool(payload.get("on")) - - -class Record(Hook): - def __init__(self): - self.events = [] - - def before(self, step, payload): - self.events.append(("before", step)) - - def after(self, step, payload): - self.events.append(("after", step)) - - def on_error(self, step, error, payload): - self.events.append(("error", step)) - - -def _pipeline(*steps): - p = Pipeline() - for step in steps: - p.add_filter(step) - return p - - -class TestIdentityIsTheFiltersSymbol: - def test_state_reports_the_qualified_symbol(self): - p = _pipeline(Add()) - p.run(Payload({})) - assert p.state.executed == [f"{MODULE}.Add"] - - def test_reuse_is_numbered_from_the_second_occurrence(self): - p = _pipeline(Add(), Add(), Add()) - p.run(Payload({})) - assert p.state.executed == [f"{MODULE}.Add", f"{MODULE}.Add#2", f"{MODULE}.Add#3"] - - def test_taps_are_identified_the_same_way(self): - p = _pipeline(Add()) - p.add_tap(Audit()) - p.run(Payload({})) - assert p.state.executed == [f"{MODULE}.Add", f"{MODULE}.Audit"] - - def test_hooks_receive_the_identity(self): - p = _pipeline(Add(), Boom()) - hook = Record() - p.use_hook(hook) - with pytest.raises(RuntimeError): - p.run(Payload({})) - assert hook.events == [("before", None), ("before", f"{MODULE}.Add"), ("after", f"{MODULE}.Add"), - ("before", f"{MODULE}.Boom"), ("error", f"{MODULE}.Boom"), ("error", None)] - - def test_a_valve_is_identified_by_its_inner_filter(self): - p = Pipeline() - p.add_valve(IsOn(), Add()) - p.run(Payload({})) - assert p.state.skipped == [f"{MODULE}.Add"] - - def test_a_valve_and_a_plain_use_of_the_same_filter_are_numbered(self): - p = _pipeline(Add()) - p.add_valve(IsOn(), Add()) - p.run(Payload({"on": True})) - assert p.state.executed == [f"{MODULE}.Add", f"{MODULE}.Add#2"] - - def test_a_named_pipeline_type_nests_as_one_step(self): - outer = _pipeline(Add()) - outer.add_pipeline(UserOnboarding()) - assert outer.run(Payload({})).get("n") == 2 - assert outer.state.executed == [f"{MODULE}.Add", f"{MODULE}.UserOnboarding"] - - def test_identities_lists_steps_in_order(self): - p = _pipeline(Add(), Add()) - assert p.identities == [f"{MODULE}.Add", f"{MODULE}.Add#2"] - - -class TestPredicateObjects: - def test_the_predicate_is_identified_by_its_class(self): - assert Valve(Add(), IsOn()).predicate_symbol == f"{MODULE}.IsOn" - - def test_a_configurable_predicate(self): - p = Pipeline() - p.add_valve(AtLeast(5), Add()) - assert p.run(Payload({"n": 5})).get("n") == 6 - assert p.run(Payload({"n": 1})).get("n") == 1 - - -class TestAddSite: - def test_the_add_call_location_is_recorded(self): - p = Pipeline() - p.add_filter(Add()) # this line - site = p.step_sites[f"{MODULE}.Add"] - assert site.endswith("test_step_identity.py:%d" % (TestAddSite.line_of_add_call(),)) - - @staticmethod - def line_of_add_call(): - import inspect - source, start = inspect.getsourcelines(TestAddSite.test_the_add_call_location_is_recorded) - return start + next(i for i, line in enumerate(source) if "# this line" in line) - - -def _make_closure_filter(): - class Local: - def call(self, payload): - return payload - return Local() - - -def _make_closure_predicate(): - def inner(payload): - return True - return inner - - -class TestAnonymousStepsAreRefused: - def test_class_defined_in_a_function(self): - with pytest.raises(AnonymousStep, match="Local"): - Pipeline().add_filter(_make_closure_filter()) # cup: allow-anonymous (exercises the refusal) - - def test_a_function_is_not_a_predicate(self): - with pytest.raises(AnonymousStep, match="predicate object"): - Valve(Add(), is_on) # cup: allow-anonymous (exercises the refusal) - - def test_lambda_predicate(self): - with pytest.raises(AnonymousStep, match="lambda"): - Valve(Add(), lambda payload: True) # cup: allow-anonymous (exercises the refusal) - - def test_closure_predicate(self): - with pytest.raises(AnonymousStep, match="inner"): - Valve(Add(), _make_closure_predicate()) # cup: allow-anonymous (exercises the refusal) - - def test_filter_defined_in_the_entry_script(self): - Main = type("Main", (), {"call": lambda self, payload: payload, "__module__": "__main__"}) - with pytest.raises(AnonymousStep, match="__main__"): - Pipeline().add_filter(Main()) - - def test_plain_pipeline_cannot_be_nested(self): - with pytest.raises(AnonymousStep, match="named Pipeline type"): - Pipeline().add_pipeline(_pipeline(Add())) # cup: allow-anonymous (exercises the refusal) - - def test_the_message_names_the_fix(self): - with pytest.raises(AnonymousStep) as info: - Pipeline().add_filter(_make_closure_filter()) # cup: allow-anonymous (exercises the refusal) - assert "module level" in str(info.value) - - def test_anonymous_step_is_a_type_error(self): - assert issubclass(AnonymousStep, TypeError) - - -class TestNamesAreGone: - def test_add_filter_takes_no_name(self): - with pytest.raises(TypeError): - Pipeline().add_filter(Add(), "add") # type: ignore[call-arg] - - def test_valve_takes_no_name(self): - with pytest.raises(TypeError): - Valve("gate", Add(), IsOn()) # type: ignore[call-arg] - - diff --git a/tests/core/test_step_labels.py b/tests/core/test_step_labels.py new file mode 100644 index 0000000..ee2a0c8 --- /dev/null +++ b/tests/core/test_step_labels.py @@ -0,0 +1,170 @@ +"""A step is reported by a label: the name of its class. + +State and Hooks use the label. Nothing is refused for its name: a class in a +script, a class in a function and a plain nested Pipeline all work. The +linter is the guard for names. +""" + +import pytest + +import codeupipe +from codeupipe import Hook, Payload, Pipeline + + +class AddFilter: + def call(self, payload): + return payload.insert("n", payload.get("n", 0) + 1) + + +class FailFilter: + def call(self, payload): + raise RuntimeError("boom") + + +class SeeTap: + def observe(self, payload): + pass + + +class IsOnPredicate: + def test(self, payload): + return payload.get("on") is True + + +class InnerPipeline(Pipeline): + def __init__(self): + super().__init__() + self.add_filter(AddFilter()) + + +class RecordHook(Hook): + def __init__(self): + self.events = [] + + def before(self, step, payload): + self.events.append(("before", step)) + + def after(self, step, payload): + self.events.append(("after", step)) + + def on_error(self, step, error, payload): + self.events.append(("error", step)) + + +def test_the_label_is_the_class_name(): + pipeline = Pipeline() + pipeline.add_filter(AddFilter()) + pipeline.add_tap(SeeTap()) + assert pipeline.labels == ["AddFilter", "SeeTap"] + + +def test_state_reports_labels(): + pipeline = Pipeline() + pipeline.add_filter(AddFilter()) + pipeline.add_tap(SeeTap()) + pipeline.run(Payload({})) + assert pipeline.state.executed == ["AddFilter", "SeeTap"] + + +def test_the_second_use_of_a_class_gets_a_number(): + pipeline = Pipeline() + for _ in range(3): + pipeline.add_filter(AddFilter()) + assert pipeline.labels == ["AddFilter", "AddFilter#2", "AddFilter#3"] + assert pipeline.run(Payload({})).get("n") == 3 + assert pipeline.state.executed == ["AddFilter", "AddFilter#2", "AddFilter#3"] + + +def test_a_valve_has_the_label_of_its_filter(): + pipeline = Pipeline() + pipeline.add_valve(IsOnPredicate(), AddFilter()) + assert pipeline.labels == ["AddFilter"] + pipeline.run(Payload({"on": False})) + assert pipeline.state.skipped == ["AddFilter"] + + +def test_a_nested_pipeline_has_the_label_of_its_class(): + outer = Pipeline() + outer.add_pipeline(InnerPipeline()) + assert outer.labels == ["InnerPipeline"] + outer.run(Payload({})) + assert outer.state.executed == ["InnerPipeline"] + + +def test_a_plain_nested_pipeline_is_accepted(): + inner = Pipeline() + inner.add_filter(AddFilter()) + outer = Pipeline() + outer.add_pipeline(inner) # cup: allow-anonymous (a plain nested Pipeline works at run time) + assert outer.labels == ["Pipeline"] + assert outer.run(Payload({})).get("n") == 1 + + +def test_hooks_receive_labels(): + hook = RecordHook() + pipeline = Pipeline() + pipeline.add_filter(AddFilter()) + pipeline.use_hook(hook) + pipeline.run(Payload({})) + assert hook.events == [("before", None), ("before", "AddFilter"), ("after", "AddFilter"), ("after", None)] + + +def test_an_error_names_the_label(): + hook = RecordHook() + pipeline = Pipeline() + pipeline.add_filter(FailFilter()) + pipeline.use_hook(hook) + with pytest.raises(RuntimeError): + pipeline.run(Payload({})) + assert ("error", "FailFilter") in hook.events + + +def test_a_class_defined_in_a_function_works(): + class LocalFilter: + def call(self, payload): + return payload.insert("local", True) + + pipeline = Pipeline() + pipeline.add_filter(LocalFilter()) # cup: allow-anonymous (a local class works at run time) + assert pipeline.labels == ["LocalFilter"] + assert pipeline.run(Payload({})).get("local") is True + + +def test_a_class_from_the_entry_script_works(): + script_class = type("ScriptFilter", (), {"__module__": "__main__", + "call": lambda self, payload: payload.insert("s", 1)}) + pipeline = Pipeline() + pipeline.add_filter(script_class()) + assert pipeline.labels == ["ScriptFilter"] + assert pipeline.run(Payload({})).get("s") == 1 + + +@pytest.mark.parametrize("add, bad, says", [ + ("add_filter", lambda payload: payload, "call(payload)"), + ("add_filter", object(), "call(payload)"), + ("add_tap", object(), "observe(payload)"), + ("add_pipeline", object(), "Pipeline"), +]) +def test_a_step_with_the_wrong_shape_is_refused_when_it_is_added(add, bad, says): + with pytest.raises(TypeError) as error: + getattr(Pipeline(), add)(bad) # cup: allow-anonymous (exercises the refusal) + assert says in str(error.value) + + +def test_a_predicate_with_no_test_method_is_refused(): + def is_on(payload): + return True + with pytest.raises(TypeError) as error: + Pipeline().add_valve(is_on, AddFilter()) # cup: allow-anonymous (exercises the refusal) + assert "test(payload)" in str(error.value) + + +def test_a_name_argument_is_not_accepted(): + with pytest.raises(TypeError): + Pipeline().add_filter(AddFilter(), "a_name") + + +def test_the_identity_names_are_gone(): + assert "AnonymousStep" not in codeupipe.__all__ + assert not hasattr(Pipeline(), "identities") + assert not hasattr(Pipeline(), "step_sites") diff --git a/tests/core/test_sync_pipeline.py b/tests/core/test_sync_pipeline.py index 1353264..e370b35 100644 --- a/tests/core/test_sync_pipeline.py +++ b/tests/core/test_sync_pipeline.py @@ -100,7 +100,7 @@ def test_state_tracks_execution(self): sp.add_filter(AddOneFilter()) sp.add_filter(UpperFilter()) sp.run(Payload({"x": 1, "name": "alice"})) - assert sp.state.executed == [f"{__name__}.AddOneFilter", f"{__name__}.UpperFilter"] + assert sp.state.executed == ["AddOneFilter", "UpperFilter"] def test_returns_payload_type(self): sp = Pipeline() @@ -129,7 +129,7 @@ def test_tap_state_tracked(self): sp = Pipeline() sp.add_tap(RecordTap()) sp.run(Payload({"x": 1})) - assert f"{__name__}.RecordTap" in sp.state.executed + assert "RecordTap" in sp.state.executed # ── Hooks ─────────────────────────────────────────────────────────── @@ -171,8 +171,8 @@ def test_timing_recorded(self): sp.add_filter(AddOneFilter()) sp.observe(timing=True) sp.run(Payload({"x": 0})) - assert f"{__name__}.AddOneFilter" in sp.state.timings - assert sp.state.timings[f"{__name__}.AddOneFilter"] > 0 + assert "AddOneFilter" in sp.state.timings + assert sp.state.timings["AddOneFilter"] > 0 # ── Govern contracts ──────────────────────────────────────────────── @@ -264,7 +264,7 @@ def test_run_contains_a_tap_failure(self): result = p.run(Payload({})) assert result.get("score") == 2 - assert p.state.errors[-1][0] == f"{__name__}.TestTapFailureIsolation._Boom" - assert f"{__name__}.TestTapFailureIsolation._Boom" in p.state.executed + assert p.state.errors[-1][0] == "_Boom" + assert "_Boom" in p.state.executed assert hook.errors == [] diff --git a/tests/core/test_sync_support.py b/tests/core/test_sync_support.py index ef0768b..5afbcea 100644 --- a/tests/core/test_sync_support.py +++ b/tests/core/test_sync_support.py @@ -71,7 +71,7 @@ async def go(): return pipeline.state state = run(go()) - assert f"{__name__}.SyncPass" in state.executed + assert "SyncPass" in state.executed # =========================================================================== @@ -128,7 +128,7 @@ async def go(): return pipeline.state state = run(go()) - assert f"{__name__}.SyncNopTap" in state.executed + assert "SyncNopTap" in state.executed # =========================================================================== @@ -167,8 +167,8 @@ async def go(): run(go()) assert log == [ "before:pipeline", - f"before:{__name__}.PassFilter", - f"after:{__name__}.PassFilter", + "before:PassFilter", + "after:PassFilter", "after:pipeline", ] @@ -192,7 +192,7 @@ async def go(): run(go()) # the failing step first, then the pipeline - assert errors == [f"{__name__}.BoomFilter:boom", "pipeline:boom"] + assert errors == ["BoomFilter:boom", "pipeline:boom"] # =========================================================================== @@ -230,7 +230,7 @@ async def go(): return pipeline.state state = run(go()) - assert f"{__name__}.SyncDiscount2" in state.skipped + assert "SyncDiscount2" in state.skipped # =========================================================================== diff --git a/tests/core/test_valve_sync.py b/tests/core/test_valve_sync.py index 9dd72f5..9fe8f63 100644 --- a/tests/core/test_valve_sync.py +++ b/tests/core/test_valve_sync.py @@ -132,7 +132,7 @@ def test_skipped_valve_is_marked_skipped(self): pipeline = Pipeline() pipeline.add_valve(_Predicate8(), Append()) pipeline.run(Payload({"s": "a"})) - assert f"{__name__}.Append" in pipeline.state.skipped + assert "Append" in pipeline.state.skipped def test_valve_is_no_longer_far_slower_than_a_plain_step(self): """The regression guard. Not a benchmark — an order-of-magnitude diff --git a/tests/examples/test_pricing.py b/tests/examples/test_pricing.py index 78f147a..dc8a72b 100644 --- a/tests/examples/test_pricing.py +++ b/tests/examples/test_pricing.py @@ -1,6 +1,6 @@ """The README example. Each primitive is in its own file, and the pipeline -file puts them together. It runs, a failing Tap does not halt it, and its -export gives the same answer with no framework import.""" +file puts them together. It runs, and its export gives the same answer +with no framework import.""" import importlib.util import pathlib @@ -10,6 +10,7 @@ from codeupipe import Payload from examples.pricing.at_least_predicate import AtLeastPredicate +from examples.pricing.audit_tap import AuditTap from examples.pricing.discount_filter import DiscountFilter from examples.pricing.mark_bulk_filter import MarkBulkFilter from examples.pricing.pricing_pipeline import PricingPipeline @@ -47,25 +48,31 @@ def test_dynamic_run(): pipeline = PricingPipeline() out = pipeline.run(Payload({"price": 100.0, "qty": 12})) assert out.to_dict() == {"price": 90.0, "qty": 12, "bulk": True} - assert pipeline.identities == ["examples.pricing.discount_filter.DiscountFilter", - "examples.pricing.mark_bulk_filter.MarkBulkFilter", - "examples.pricing.audit_tap.AuditTap"] - assert [name for name, _ in pipeline.state.errors] == ["examples.pricing.audit_tap.AuditTap"] + assert pipeline.labels == ["DiscountFilter", + "MarkBulkFilter", + "AuditTap"] + assert pipeline.state.executed == ["DiscountFilter", "MarkBulkFilter", "AuditTap"] + assert pipeline.state.errors == [] def test_valve_skips_small_orders(): pipeline = PricingPipeline() out = pipeline.run(Payload({"price": 100.0, "qty": 2})) assert out.to_dict() == {"price": 90.0, "qty": 2} - assert pipeline.state.skipped == ["examples.pricing.mark_bulk_filter.MarkBulkFilter"] + assert pipeline.state.skipped == ["MarkBulkFilter"] -def test_a_hook_sees_each_step(capsys): +def test_the_tap_reads_the_payload(capsys): + assert AuditTap().observe(Payload({"price": 90.0})) is None + assert capsys.readouterr().out == "audit: {'price': 90.0}\n" + + +def test_a_hook_sees_each_step_but_not_the_tap(capsys): pipeline = PricingPipeline() pipeline.use_hook(PrintStepsHook()) pipeline.run(Payload({"price": 100.0, "qty": 12})) lines = capsys.readouterr().out.splitlines() - assert lines == [" discount_filter.DiscountFilter", " mark_bulk_filter.MarkBulkFilter"] + assert lines == [" DiscountFilter", " MarkBulkFilter", "audit: {'price': 90.0, 'qty': 12, 'bulk': True}"] def test_the_example_runs_as_a_module(): @@ -77,7 +84,7 @@ def test_the_example_runs_as_a_module(): def test_export_matches_and_has_no_framework_import(tmp_path): done = subprocess.run( - [sys.executable, "-m", "codeupipe.lower", "examples.pricing.pricing_pipeline:PricingPipeline", "--out", str(tmp_path)], + [sys.executable, "-m", "codeupipe.compile", "examples.pricing.pricing_pipeline:PricingPipeline", "--out", str(tmp_path)], capture_output=True, text=True, cwd=EXAMPLE.parents[1], ) assert done.returncode == 0, done.stderr diff --git a/tests/lint/anonymous_steps/test_analyze_python_filter.py b/tests/lint/anonymous_steps/test_analyze_python_filter.py index 7c49c6c..9d31963 100644 --- a/tests/lint/anonymous_steps/test_analyze_python_filter.py +++ b/tests/lint/anonymous_steps/test_analyze_python_filter.py @@ -10,7 +10,7 @@ from ._support import by_line, expectations, fixture, run_analyzer -@pytest.mark.parametrize("name", ["positive.py", "negative.py", "entry_script.py", "probe.py"]) +@pytest.mark.parametrize("name", ["positive.py", "negative.py", "probe.py"]) def test_fixture_findings_match_expectations(name): path = fixture("python", name) findings, errors = run_analyzer(AnalyzePythonFilter(), "python", [path]) diff --git a/tests/lint/anonymous_steps/test_lint_pipeline.py b/tests/lint/anonymous_steps/test_lint_pipeline.py index 0c8e01e..9727cb3 100644 --- a/tests/lint/anonymous_steps/test_lint_pipeline.py +++ b/tests/lint/anonymous_steps/test_lint_pipeline.py @@ -6,11 +6,11 @@ def test_the_linter_is_a_pipeline_of_named_filters(): - identities = build_lint_pipeline().identities - assert identities[0] == "codeupipe_lint.anonymous_steps.collect_files_filter.CollectFilesFilter" - assert "codeupipe_lint.anonymous_steps.analyze_python_filter.AnalyzePythonFilter" in identities - assert identities[-2:] == ["codeupipe_lint.anonymous_steps.apply_suppressions_filter.ApplySuppressionsFilter", - "codeupipe_lint.anonymous_steps.format_report_filter.FormatReportFilter"] + labels = build_lint_pipeline().labels + assert labels[0] == "CollectFilesFilter" + assert "AnalyzePythonFilter" in labels + assert labels[-2:] == ["ApplySuppressionsFilter", + "FormatReportFilter"] def test_lint_paths_reports_and_sets_the_exit_code(): diff --git a/tests/lint/anonymous_steps/test_repo_is_clean.py b/tests/lint/anonymous_steps/test_repo_is_clean.py index aee12eb..6eb2d6e 100644 --- a/tests/lint/anonymous_steps/test_repo_is_clean.py +++ b/tests/lint/anonymous_steps/test_repo_is_clean.py @@ -7,10 +7,10 @@ from ._support import REPO -CHECKED = ["packages/codeupipe/codeupipe", "conformance/runners/python", "scorecard", "tests/core", "tests/lower", +CHECKED = ["packages/codeupipe/codeupipe", "conformance/runners/python", "scorecard", "tests/core", "tests/compile", "tests/scorecard", "tests/conformance"] -INTENTIONAL = {"tests/core/test_step_identity.py", "tests/lower/test_export_refusals.py", - "tests/core/test_add_valve.py"} +INTENTIONAL = {"tests/compile/test_export_refusals.py", + "tests/core/test_add_valve.py", "tests/core/test_step_labels.py"} def test_repo_python_has_no_anonymous_steps(): diff --git a/tests/lint/fixtures/python/entry_script.py b/tests/lint/fixtures/python/entry_script.py deleted file mode 100644 index cb0fa57..0000000 --- a/tests/lint/fixtures/python/entry_script.py +++ /dev/null @@ -1,38 +0,0 @@ -"""A script: its classes are __main__.X when run, so steps defined here and -added from the __main__ block (directly or via a function it calls) are flagged.""" - -from codeupipe import Payload, Pipeline - - -class Local: - def call(self, payload): - return payload - - -def is_on(payload): - return True - - -def library_builder(): - # Not reached from the __main__ block: an importer gets a real module name. - p = Pipeline() - p.add_filter(Local()) - return p - - -def main(): - p = Pipeline() - p.add_filter(Local()) # expect: CUP-A006 - p.add_valve(is_on, Local()) # expect: CUP-A006 CUP-A006 - helper(p) - return p.run(Payload({})) - - -def helper(p): - p.add_tap(Local()) # expect: CUP-A006 - - -if __name__ == "__main__": - main() - q = Pipeline() - q.add_filter(Local()) # expect: CUP-A006 diff --git a/tests/lint/fixtures/python/probe.py b/tests/lint/fixtures/python/probe.py index e67532f..fbdf9e2 100644 --- a/tests/lint/fixtures/python/probe.py +++ b/tests/lint/fixtures/python/probe.py @@ -1,4 +1,4 @@ -"""Step-identity probe cases, with describe(label, f) replaced by +"""Step probe cases, with describe(label, f) replaced by p.add_filter(f): the probe's cases as lint fixtures.""" from codeupipe import Pipeline @@ -31,4 +31,4 @@ def call(self, payload): p.add_filter(lambda payload: payload) # expect: CUP-A001 p.add_filter(make_closure("x")) # expect: CUP-A002 p.add_filter(make_local_class()) # expect: CUP-A003 -p.add_filter(DefinedInEntryScript()) # not flagged: no __main__ guard, so the linter stays conservative (the runtime refuses it) +p.add_filter(DefinedInEntryScript()) # not flagged: a module-level class diff --git a/tests/lint/invariants/test_plain_logic.py b/tests/lint/invariants/test_plain_logic.py new file mode 100644 index 0000000..afcb78f --- /dev/null +++ b/tests/lint/invariants/test_plain_logic.py @@ -0,0 +1,79 @@ +"""CUP-I008: logic that is not in a primitive. A warning.""" + +from codeupipe_lint.invariants.lint_pipeline import lint_paths +from codeupipe_lint.invariants.__main__ import main + + +def lint(tmp_path, files, **options): + for name, text in files.items(): + (tmp_path / name).write_text(text) + out = lint_paths([str(tmp_path)], **options) + found = ["%s:%d %s" % (f.path[len(str(tmp_path)) + 1:], f.line, f.rule) for f in out.get("findings")] + return out, found + + +def test_a_function_with_a_decision_that_is_not_in_a_primitive_is_a_warning(tmp_path): + out, found = lint(tmp_path, {"who_is_due.py": ( + 'def who_is_due(entries):\n if not entries:\n return "say"\n return "model"\n' + "def pick(items):\n for each in items:\n return each\n" + 'def choose(flag):\n return "a" if flag else "b"\n' + "async def load(path):\n try:\n return await read(path)\n except OSError:\n return None\n" + "def wait(queue):\n while queue:\n queue.pop()\n" + "def plain(a, b):\n return a + b\n" + "def evens(items):\n return [each for each in items if each % 2 == 0]\n")}) + assert found == ["who_is_due.py:1 CUP-I008", "who_is_due.py:5 CUP-I008", "who_is_due.py:8 CUP-I008", + "who_is_due.py:10 CUP-I008", "who_is_due.py:15 CUP-I008"] + first = out.get("findings")[0] + assert first.severity == "warning" + assert first.message.startswith("Function `who_is_due` has logic and is not a Filter, a Predicate, a Tap, " + "a Hook or a Pipeline. Write it as a primitive") + assert "CUP-I008 logic-outside-primitive warning: Function `who_is_due`" in out.get("report") + + +def test_a_class_that_is_not_a_primitive_and_code_at_the_top_of_a_file(tmp_path): + _, found = lint(tmp_path, { + "record_store.py": ( + "class RecordStore:\n def read(self, name):\n if name not in self.files:\n" + " raise KeyError(name)\n return []\n" + "class Point:\n def __init__(self, x):\n self.x = x\n"), + "main.py": "import sys\nARGS = sys.argv[1:]\nif not ARGS:\n sys.exit(2)\nfor arg in ARGS:\n print(arg)\n"}) + assert found == ["main.py:3 CUP-I008", "record_store.py:1 CUP-I008"] + + +def test_logic_in_a_primitive_is_where_it_must_be(tmp_path): + _, found = lint(tmp_path, { + "pick_filter.py": ("class PickFilter:\n def call(self, payload):\n if payload.get('a'):\n" + " return payload.insert('b', 1)\n return payload\n"), + "is_on_predicate.py": "class IsOnPredicate:\n def test(self, payload):\n return True if payload.get('on') else False\n", + "order_pipeline.py": ("from codeupipe import Pipeline\nclass OrderPipeline(Pipeline):\n def __init__(self):\n" + " super().__init__()\n for step in ():\n self.add_filter(step)\n")}) + assert found == [] + + +def test_the_usual_guards_at_the_top_of_a_file_are_not_logic(tmp_path): + _, found = lint(tmp_path, {"entry.py": ( + "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n import ast\n" + "def main():\n return 0\n" + 'if __name__ == "__main__":\n raise SystemExit(main())\n')}) + assert found == [] + + +def test_a_test_file_is_not_checked(tmp_path): + _, found = lint(tmp_path, {"test_pick.py": "def test_pick():\n for each in ():\n assert each\n"}) + assert found == [] + + +def test_a_comment_allows_plain_logic(tmp_path): + out, found = lint(tmp_path, {"shell.py": ( + "# cup: allow-plain the edge to the shell\n" + "def run(command):\n try:\n return start(command)\n except OSError:\n return None\n")}) + assert found == [] + assert len(out.get("suppressed")) == 1 + + +def test_a_warning_fails_the_run_only_when_it_is_strict(tmp_path, capsys): + (tmp_path / "pick.py").write_text("def pick(flag):\n return 1 if flag else 2\n") + assert main([str(tmp_path)]) == 0 + assert "No invariant violations. 1 warning." in capsys.readouterr().out + assert main(["--strict", str(tmp_path)]) == 1 + assert main(["--strict", "--disable", "CUP-I008", str(tmp_path)]) == 0 diff --git a/tests/lint/invariants/test_repo_is_clean.py b/tests/lint/invariants/test_repo_is_clean.py index f2a1568..b610704 100644 --- a/tests/lint/invariants/test_repo_is_clean.py +++ b/tests/lint/invariants/test_repo_is_clean.py @@ -1,16 +1,30 @@ """This repo's own Python keeps the invariants it lints for.""" import os +import pathlib from codeupipe_lint.invariants.lint_pipeline import lint_paths from ._support import REPO +# The places with a `cup: allow-plain` comment. +PLAIN_BY_DESIGN = 63 + CHECKED = ["packages/codeupipe/codeupipe", "conformance", "scorecard", "packages/codeupipe-lint/codeupipe_lint", "tests", "examples"] def test_repo_python_has_no_invariant_violations(): out = lint_paths([os.path.join(REPO, p) for p in CHECKED], exclude=["*/tests/lint/fixtures/*"]) assert out.get("errors") == [] + # No finding and no warning: the repository passes its own linter in strict mode. assert [f.format() for f in out.get("findings")] == [] - assert out.get("suppressed") == [] + # The only comments that allow a finding are for CUP-I008, on code that is plain for a reason that + # the comment gives: a primitive itself, a visitor of a syntax tree, the entry of a program, a test + # harness. The number is here so that a new one is a change that someone sees. + allowed = out.get("suppressed") + plain = [f for f in allowed if f.rule == "CUP-I008"] + assert len(plain) == PLAIN_BY_DESIGN, sorted({f.path for f in plain}) + # One more comment, for another rule: the conformance runner reads the State of the pipeline of a + # case, because that State is what it reports. + other = [(f.rule, pathlib.Path(f.path).name) for f in allowed if f.rule != "CUP-I008"] + assert other == [("CUP-I003", "run_dynamic_filter.py")] diff --git a/tests/lint/invariants/test_stale_markers.py b/tests/lint/invariants/test_stale_markers.py new file mode 100644 index 0000000..b2929c2 --- /dev/null +++ b/tests/lint/invariants/test_stale_markers.py @@ -0,0 +1,45 @@ +"""CUP-I009: a comment that allows nothing. A warning.""" + +from codeupipe_lint.invariants.__main__ import main + +from .test_plain_logic import lint + + +def test_a_comment_that_allows_nothing_is_a_warning(tmp_path): + out, found = lint(tmp_path, {"add.py": ( + "# cup: allow-plain it had a loop before\n" + "def add(a, b):\n return a + b\n" + "ONE = 1 # cup: allow-state-read nothing here reads State\n" + "# cup: allow-plian a kind that does not exist\n" + "TWO = 2\n")}) + assert found == ["add.py:1 CUP-I009", "add.py:4 CUP-I009", "add.py:5 CUP-I009"] + first = out.get("findings")[0] + assert first.severity == "warning" + assert first.message.startswith("The comment `cup: allow-plain` allows nothing. Remove it") + assert out.get("exit_code") == 0 + + +def test_a_stale_comment_fails_a_strict_run(tmp_path): + (tmp_path / "one.py").write_text("# cup: allow-plain no reason\nONE = 1\n") + assert main(["--strict", str(tmp_path)]) == 1 + assert main(["--strict", "--disable", "CUP-I009", str(tmp_path)]) == 0 + + +def test_a_comment_that_allows_a_finding_is_not_reported_and_a_string_is_not_a_comment(tmp_path): + out, found = lint(tmp_path, {"pick.py": ( + "# cup: allow-plain the entry of the program\n" + "def pick(items):\n for each in items:\n return each\n" + "def choose(flag): # cup: allow-plain a small helper\n return 1 if flag else 2\n" + '"""The marker ``# cup: allow-plain`` allows a finding."""\n' + 'HELP = "say why: # cup: allow-plain "\n' + "# For plain logic, write cup: allow-plain and the reason.\n")}) + assert found == [] + assert len(out.get("suppressed")) == 2 + + +def test_a_comment_for_a_rule_that_is_off_or_for_the_other_linter_is_not_reported(tmp_path): + out, found = lint(tmp_path, {"pick.py": ( + "# cup: allow-plain the entry\ndef pick(flag):\n return 1 if flag else 2\n" + "STEP = 1 # cup: allow-anonymous the other linter reads this one\n")}, disable=["CUP-I008"]) + assert found == [] + assert out.get("suppressed") == []