From 8920de256f0b9e15d2f2496442088c8aad852039 Mon Sep 17 00:00:00 2001 From: Joshua Wink Date: Wed, 28 Oct 2026 22:53:19 -0500 Subject: [PATCH 1/2] feat(rust): codeupipe-compile, a pipeline written as plain Rust packages/codeupipe-compile-rs: codeupipe-compile reads a Rust pipeline and its steps with syn, and writes them as a module of plain Rust: a struct with one Option field for each key that the steps use, one function for each step, and a run function. The body of each step is its own code, with each key of the Payload as a field and each setting as its literal. It does not run the code. The rule: code that compiles gives the same output as the pipeline. A pipeline for which the compiler cannot keep that rule is refused, with the place and the fix: a Hook, a nested pipeline, a setting that is not a literal, a key that is not text in the code, a key with two types, a read with no type, another method of the Payload, and others. The compiler is a codeupipe pipeline of five Filters. tests/same_output.rs compiles the expected modules into the test and compares them with the pipelines on many inputs and errors. On an M1, pricing (3 steps of arithmetic) runs 150x faster than Pipeline::run, signup (4 steps of text) 3.1x. A Cargo workspace of its own, as the linter. A new CI job runs fmt, clippy, the tests, the bench and Rust 1.75; the linter job also checks its code. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 24 ++ .gitignore | 1 + README.md | 4 +- conformance/API.md | 2 +- docs/rust-plan.md | 15 +- packages/codeupipe-compile-rs/Cargo.lock | 63 +++ packages/codeupipe-compile-rs/Cargo.toml | 28 ++ packages/codeupipe-compile-rs/LICENSE | 177 ++++++++ packages/codeupipe-compile-rs/NOTICE | 4 + packages/codeupipe-compile-rs/README.md | 172 ++++++++ .../codeupipe-compile-rs/examples/bench.rs | 86 ++++ .../src/compile_pipeline.rs | 21 + .../codeupipe-compile-rs/src/compile_run.rs | 35 ++ .../src/emit_module_filter.rs | 81 ++++ packages/codeupipe-compile-rs/src/key_type.rs | 31 ++ packages/codeupipe-compile-rs/src/key_use.rs | 15 + packages/codeupipe-compile-rs/src/lib.rs | 52 +++ .../src/lower_steps_filter.rs | 184 +++++++++ .../codeupipe-compile-rs/src/lowered_step.rs | 10 + packages/codeupipe-compile-rs/src/main.rs | 53 +++ .../src/payload_rewriter.rs | 348 ++++++++++++++++ packages/codeupipe-compile-rs/src/place.rs | 20 + .../src/read_sources_filter.rs | 63 +++ packages/codeupipe-compile-rs/src/refusal.rs | 32 ++ .../src/resolve_pipeline_filter.rs | 390 ++++++++++++++++++ packages/codeupipe-compile-rs/src/source.rs | 6 + packages/codeupipe-compile-rs/src/step.rs | 30 ++ .../src/type_keys_filter.rs | 100 +++++ .../codeupipe-compile-rs/tests/compile.rs | 170 ++++++++ .../tests/expected/pricing.rs | 49 +++ .../tests/expected/signup.rs | 65 +++ .../fixtures/pricing/at_least_predicate.rs | 25 ++ .../tests/fixtures/pricing/audit_tap.rs | 13 + .../tests/fixtures/pricing/discount_filter.rs | 19 + .../fixtures/pricing/mark_bulk_filter.rs | 10 + .../tests/fixtures/pricing/mod.rs | 5 + .../fixtures/pricing/pricing_pipeline.rs | 16 + .../fixtures/signup/apply_coupon_filter.rs | 20 + .../tests/fixtures/signup/count_tap.rs | 11 + .../fixtures/signup/has_key_predicate.rs | 17 + .../tests/fixtures/signup/mod.rs | 6 + .../fixtures/signup/normalize_email_filter.rs | 20 + .../tests/fixtures/signup/signup_pipeline.rs | 18 + .../tests/fixtures/signup/welcome_filter.rs | 21 + .../codeupipe-compile-rs/tests/same_output.rs | 98 +++++ 45 files changed, 2627 insertions(+), 3 deletions(-) create mode 100644 packages/codeupipe-compile-rs/Cargo.lock create mode 100644 packages/codeupipe-compile-rs/Cargo.toml create mode 100644 packages/codeupipe-compile-rs/LICENSE create mode 100644 packages/codeupipe-compile-rs/NOTICE create mode 100644 packages/codeupipe-compile-rs/README.md create mode 100644 packages/codeupipe-compile-rs/examples/bench.rs create mode 100644 packages/codeupipe-compile-rs/src/compile_pipeline.rs create mode 100644 packages/codeupipe-compile-rs/src/compile_run.rs create mode 100644 packages/codeupipe-compile-rs/src/emit_module_filter.rs create mode 100644 packages/codeupipe-compile-rs/src/key_type.rs create mode 100644 packages/codeupipe-compile-rs/src/key_use.rs create mode 100644 packages/codeupipe-compile-rs/src/lib.rs create mode 100644 packages/codeupipe-compile-rs/src/lower_steps_filter.rs create mode 100644 packages/codeupipe-compile-rs/src/lowered_step.rs create mode 100644 packages/codeupipe-compile-rs/src/main.rs create mode 100644 packages/codeupipe-compile-rs/src/payload_rewriter.rs create mode 100644 packages/codeupipe-compile-rs/src/place.rs create mode 100644 packages/codeupipe-compile-rs/src/read_sources_filter.rs create mode 100644 packages/codeupipe-compile-rs/src/refusal.rs create mode 100644 packages/codeupipe-compile-rs/src/resolve_pipeline_filter.rs create mode 100644 packages/codeupipe-compile-rs/src/source.rs create mode 100644 packages/codeupipe-compile-rs/src/step.rs create mode 100644 packages/codeupipe-compile-rs/src/type_keys_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/compile.rs create mode 100644 packages/codeupipe-compile-rs/tests/expected/pricing.rs create mode 100644 packages/codeupipe-compile-rs/tests/expected/signup.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/at_least_predicate.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/audit_tap.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/discount_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/mark_bulk_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/mod.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/pricing/pricing_pipeline.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/apply_coupon_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/count_tap.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/has_key_predicate.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/mod.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/normalize_email_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/signup_pipeline.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/signup/welcome_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/same_output.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de79c58..f39455c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,30 @@ jobs: run: >- cargo run --locked -q -- --strict src tests ../codeupipe-rs/src ../codeupipe-rs/examples ../codeupipe-rs/tests ../../conformance/runners/rust/src + ../codeupipe-compile-rs/src ../codeupipe-compile-rs/tests ../codeupipe-compile-rs/examples + - name: The oldest Rust that the crate supports (rust-version) + run: | + rustup toolchain install 1.75 --profile minimal + cargo +1.75 test --locked + + rust-compile: + name: Rust compiler + runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/codeupipe-compile-rs + steps: + - uses: actions/checkout@v5 + # A workspace of its own: it downloads syn, quote and prettyplease + # from crates.io, at the versions in its Cargo.lock. + - name: Format + run: cargo fmt --check + - name: Clippy + run: cargo clippy --locked --all-targets -- -D warnings + - name: Tests + run: cargo test --locked + - name: Compiled against the pipeline (numbers only) + run: cargo run --locked --release -q --example bench - name: The oldest Rust that the crate supports (rust-version) run: | rustup toolchain install 1.75 --profile minimal diff --git a/.gitignore b/.gitignore index 003e60c..86da32a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ node_modules/ /target/ /packages/codeupipe-lint-rs/target/ +/packages/codeupipe-compile-rs/target/ diff --git a/README.md b/README.md index dbf14d7..b515675 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,9 @@ This repository has seven packages. | `codeupipe` for TypeScript | The eight primitives. No dependencies | [packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) | | `codeupipe-lint` for TypeScript | A linter for pipeline code in TypeScript | [packages/codeupipe-lint-ts/README.md](packages/codeupipe-lint-ts/README.md) | | `codeupipe-compile` for TypeScript | Writes a pipeline as plain TypeScript with no import of codeupipe | [packages/codeupipe-compile-ts/README.md](packages/codeupipe-compile-ts/README.md) | -| `codeupipe` for Rust | The primitives, 0.1.0. No dependencies. No compiler yet | [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) | +| `codeupipe` for Rust | The primitives, 0.1.0. No dependencies | [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) | | `codeupipe-lint` for Rust | A linter for pipeline code in Rust, for the rules that the compiler does not check | [packages/codeupipe-lint-rs/README.md](packages/codeupipe-lint-rs/README.md) | +| `codeupipe-compile` for Rust | Writes a pipeline as plain Rust: a struct and functions, with no codeupipe | [packages/codeupipe-compile-rs/README.md](packages/codeupipe-compile-rs/README.md) | ## Install @@ -208,6 +209,7 @@ An editable install needs pip 21.3 or later. | `packages/codeupipe-compile-ts/` | The TypeScript compiler, with its tests | | `packages/codeupipe-rs/` | The Rust runtime, with its tests and its examples | | `packages/codeupipe-lint-rs/` | The Rust linter, with its tests. A Cargo workspace of its own | +| `packages/codeupipe-compile-rs/` | The Rust compiler, with its tests. A Cargo workspace of its own | | `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runners, the benchmark | | `tests/` | The tests for the two Python packages | | `examples/pricing/` | The example, with its test in `tests/examples/` | diff --git a/conformance/API.md b/conformance/API.md index a822452..83850b5 100644 --- a/conformance/API.md +++ b/conformance/API.md @@ -242,7 +242,7 @@ map and fails the comparison. | Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | `run` is sync, and `run_with_state` for the caller's State. No async run yet | | Absent | `None` | `null` | `None` from `get`; `Value::Null` is a JSON null | | Label of a step | `type(step).__name__` | `step.constructor.name` | The name of the step's type, from `std::any::type_name` | -| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | None yet | +| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | `packages/codeupipe-compile-rs`: a struct with one field for each key. No conformance target yet | | Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | None yet. The compiler refuses several of the shapes that the linters find | | A type of your own | Any object with `copy()` | `class Order implements PipelineData`, then `new Pipeline()`, `Filter` | `impl PipelineData for Order {}`, then `Pipeline`, `Filter` | diff --git a/docs/rust-plan.md b/docs/rust-plan.md index df83672..478e2f6 100644 --- a/docs/rust-plan.md +++ b/docs/rust-plan.md @@ -432,6 +432,19 @@ core, a dynamic step costs about 25 ns, and the dynamic run is faster than compiled Python in four of the five scenarios. The compiler waits until a real pipeline shows a need. +**Started (2026-10-28), as a struct and not a value type.** Hand-compiled +forms were measured against `Pipeline::run`, on the five scenarios, the +pricing example and a step of real text work. Plain functions on a Payload +(the shape that this plan first had in mind, with a `Value` in the module) +gain 1.1 to 1.7 times: about 10 ns a step. A static pipeline from generics +reaches the same with no compiler. The large gain is the data: with a struct +that has one field for each key, the pricing example runs in about 2 ns, and +work on text 3 times faster. So the first version of the compiler writes a +struct, and keeps the rest of this section's rules: the same output, a +refusal with the place, a codeupipe pipeline itself. It is in +`packages/codeupipe-compile-rs/` ([README](../packages/codeupipe-compile-rs/README.md)). +Not in it yet: the conformance target `rust:static`, and nested pipelines. + ### The value type in compiled code Plain Python has `dict`, and plain TypeScript has `Map`. Plain Rust has no @@ -537,7 +550,7 @@ correctness. The others are about form. | 0 | Settle D1 to D10 | One review of this page | | 1 | Core crate, example, README | About 600 lines with tests (TypeScript: 415 lines of source) | | 2 | Runner, `run.py`, bench, CI, documents | About 400 lines, most of it the JSON reader and the `cup_ops` | -| 3 | Compiler and `rust:static`, if the benchmark says so | About 1,000 lines (TypeScript: 851 lines of source) | +| 3 | Compiler and `rust:static`. **Started**: the compiler, as a struct; `rust:static` not yet | About 1,000 lines (TypeScript: 851 lines of source). Written: 1,471 of source, 268 of tests | | 4 | Linter. **Done** | About 1,200 lines (written: 2,060 of source with its comments, 950 of tests) | Phases 1 and 2 are one release: `codeupipe` 0.1.0 for Rust, in the diff --git a/packages/codeupipe-compile-rs/Cargo.lock b/packages/codeupipe-compile-rs/Cargo.lock new file mode 100644 index 0000000..9d67593 --- /dev/null +++ b/packages/codeupipe-compile-rs/Cargo.lock @@ -0,0 +1,63 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "codeupipe" +version = "0.1.0" + +[[package]] +name = "codeupipe-compile" +version = "0.1.0" +dependencies = [ + "codeupipe", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" diff --git a/packages/codeupipe-compile-rs/Cargo.toml b/packages/codeupipe-compile-rs/Cargo.toml new file mode 100644 index 0000000..f888291 --- /dev/null +++ b/packages/codeupipe-compile-rs/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "codeupipe-compile" +version = "0.1.0" +edition = "2021" +rust-version = "1.75" +description = "Writes a codeupipe pipeline in Rust as plain Rust: a struct and functions, with no codeupipe." +license = "Apache-2.0" +repository = "https://github.com/orchestrate-solutions/codeupipe" +readme = "README.md" +keywords = ["pipeline", "filter", "payload", "compiler"] +# Not on crates.io yet. A version below 1.0 is not stable. +publish = false + +[[bin]] +name = "codeupipe-compile" +path = "src/main.rs" + +[dependencies] +codeupipe = { path = "../codeupipe-rs" } +# Lines and columns of the syntax tree, outside a procedural macro. +proc-macro2 = { version = "1.0.80", default-features = false, features = ["span-locations"] } +quote = { version = "1.0.35", default-features = false } +syn = { version = "2.0.60", default-features = false, features = ["full", "parsing", "printing", "clone-impls", "visit", "visit-mut"] } +prettyplease = "0.2.20" + +# A workspace of its own, as the linter: the crates of the root workspace +# have no dependencies and build with no download. +[workspace] diff --git a/packages/codeupipe-compile-rs/LICENSE b/packages/codeupipe-compile-rs/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-compile-rs/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-compile-rs/NOTICE b/packages/codeupipe-compile-rs/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-compile-rs/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-compile-rs/README.md b/packages/codeupipe-compile-rs/README.md new file mode 100644 index 0000000..c1c45c7 --- /dev/null +++ b/packages/codeupipe-compile-rs/README.md @@ -0,0 +1,172 @@ +# codeupipe-compile for Rust + +Writes a codeupipe pipeline in Rust as plain Rust: a struct with one field +for each key that the steps use, and one function for each step, with no +codeupipe. It reads the source and does not run it. + +**The rule: code that compiles gives the same output as the pipeline. A +pipeline for which the compiler cannot keep that rule is refused, with the +place named.** A pipeline that compiles and then behaves in a different way +is a bug. + +The compiler is a codeupipe pipeline: +[src/compile_pipeline.rs](src/compile_pipeline.rs). It depends on +`codeupipe`, on `syn` to read Rust, and on `prettyplease` to write it. + +The version is below 1.0: the package is not stable yet. It is not on +crates.io. Build it from this repository: + +```bash +cargo install --path packages/codeupipe-compile-rs +codeupipe-compile src/pricing PricingPipeline --out src/pricing_compiled.rs +``` + +The paths are the files (or directories) of the pipeline and its steps; the +last argument is the name of the pipeline type. With no `--out`, the module +goes to stdout. Exit code: **0** the module is written, **1** the pipeline +is refused (the message names the place and the fix), **2** the command +could not run. Nothing is written on 1 or 2. + +From Rust: `codeupipe_compile::compile(&paths, "PricingPipeline")` returns +the text of the module, or the `Refusal`. + +## Example + +The pricing pipeline of [tests/fixtures/pricing/](tests/fixtures/pricing/) +compiles to [tests/expected/pricing.rs](tests/expected/pricing.rs): + +```rust +pub struct PricingData { + pub price: Option, + pub qty: Option, + pub bulk: Option, +} + +fn cup_01_discount_filter(payload: &mut PricingData) -> Result<(), StepError> { + let price = payload.price.unwrap_or(0.0) * (1.0 - 0.1); + payload.price = Some(::core::convert::Into::into((price * 100.0).round() / 100.0)); + Ok(()) +} + +// ... one function for each step ... + +pub fn run(input: &PricingData) -> Result { + let mut payload = input.clone(); + cup_01_discount_filter(&mut payload)?; + if cup_02_when_at_least_predicate(&payload) { + cup_02_mark_bulk_filter(&mut payload)?; + } + let _ = cup_03_audit_tap(&payload); + Ok(payload) +} +``` + +`run` takes the struct and returns a new one. The struct that the caller +gives is not changed. The body of each step is its own code, with each key +of the Payload as a field and each setting as its literal, so the Rust +compiler sees all of it. + +## What compiles + +| Shape | Plain form | +|---|---| +| `impl NamedPipeline for P` (of `Payload`) | `pub fn run(input: &PData)`. `add_steps` only adds steps (a `let` for a setting is allowed; the setting must still be a literal) | +| Filter (`call`) | A function on `&mut PData` | +| Valve | `if predicate(&payload) { filter(&mut payload)?; }` | +| Tap (`observe`) | `let _ = tap(&payload);`: a failure does not stop the run | +| A step made as `T`, `T { field: literal }` or `T::new(literal, ...)` | Each field of the step is a constant, from the literal. `new` may keep a parameter as it is, or with `.to_string()`, `.into()`, `.to_owned()` or `String::from(..)` | +| `payload.get(K).and_then(Value::as_f64)` (or `as_bool`, `as_str`, or a closure `\|v\| v.as_f64()`) | `payload.k` (`payload.k.as_deref()` for text) | +| `payload.has(K)` | `payload.k.is_some()` | +| `payload.insert(K, v)` | `payload.k = Some(v.into())` | +| `payload.remove(K)` | `payload.k = None` | +| The arguments of `format!`, `println!` and other macros | Read and rewritten as code | +| `use` items of a step file that are not of codeupipe or of the project | The same `use`, in the module | + +A key `K` is text in the code: a literal, or a setting that the pipeline +gives as text. Its field has the name of the key; a character that a Rust +name cannot have becomes `_`. + +**The type of a key** comes from the reads and the writes that show one: +`as_f64` or a number is `f64`, `as_bool` or `true`/`false` is `bool`, +`as_str`, text, `format!`, `String::from` or `.to_string()` is `String`. A +key that the steps only test (`has`) or remove is text. Each field is an +`Option`, because a Payload can lack a key. + +## What is refused + +| Shape | What to do | +|---|---| +| A Hook, or `observe(...)` on the pipeline | The compiled code has no lifecycle and no State. Run the pipeline itself | +| A nested pipeline (`add_pipeline`) | Not compiled yet. Add its steps to this pipeline | +| A pipeline of a type of your own (`NamedPipeline`) | It is plain data already | +| A setting that is not a literal, or that the constructor computes | Give each setting as a literal, and compute in the method of the step | +| A key that is not text in the code | Write the key, or give it as a setting | +| A key with two types | Give each key one type | +| A key whose type no use shows (only written with an expression) | Read it once with `.and_then(Value::as_...)`, or write a literal | +| `payload.get(K)` with no `.and_then(Value::as_...)` | Read the key with its type | +| Another method of the Payload (`to_dict`, `copy`, `merge`...), or the Payload given to other code | Use the Payload only through get, has, insert and remove | +| A chain of writes (`payload.insert(..).insert(..)`) | Write one key in each statement | +| `self.method()`, or `self` used in another way | Put the code of the step in its method | +| A `Value` used in another way | Read the key with its type | +| A step with no source in the paths | Give the file of each step | + +The tests for each refusal are in [tests/compile.rs](tests/compile.rs). The +runtime's own pricing example is refused, because its Tap prints +`payload.to_dict()`, which plain data has not: + +``` +../codeupipe-rs/examples/pricing/audit_tap.rs:9:33 `payload.to_dict()` is not in plain data. Use the Payload only through get(..).and_then(Value::as_...), has, insert and remove +``` + +## What the compiled code does not have + +- **State.** No `executed`, `skipped`, `errors` or `timings`. +- **Hooks.** A pipeline with a Hook is refused. +- **Tap errors.** They are contained, as in the pipeline, but not recorded. +- **Labels.** A function has a name such as `cup_01_discount_filter`. +- **A Payload.** The caller gives and gets the struct. + +## How fast + +`cargo run --release --example bench` (Apple M1, best of 9 × 20,000, the +two alternated): + +| Pipeline | `Pipeline::run` | Compiled | | +|---|---|---|---| +| pricing: 3 steps of arithmetic | 0.237 µs | 0.0016 µs | 150x | +| signup: 4 steps of text (trim, lower case, format) | 0.504 µs | 0.164 µs | 3.1x | + +The more a step does, the less the pipeline costs next to it: with steps +of real work the gain is small. The large gain is for many runs of light +steps. + +## How it is checked + +- [tests/same_output.rs](tests/same_output.rs): the modules of + [tests/expected/](tests/expected/) are compiled into the test and give + the output and the errors of the pipelines that they come from, on many + inputs, and do not change their input. +- [tests/compile.rs](tests/compile.rs): the compiler writes those modules + (the code is compared, not its layout; `UPDATE_EXPECTED=1` writes them + again), and refuses each shape above with its place. + +## Not yet + +- A conformance target `rust:static`, judged by the cases of + `conformance/cases/` as `typescript:static` is. +- Nested pipelines. +- A `from_map`/`to_map` for a caller that has a Payload. + +## Development + +A Cargo workspace of its own, as the linter. In this directory: + +```bash +cargo test +cargo clippy --all-targets -- -D warnings +cargo run --release --example bench +``` + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-compile-rs/examples/bench.rs b/packages/codeupipe-compile-rs/examples/bench.rs new file mode 100644 index 0000000..56fb77d --- /dev/null +++ b/packages/codeupipe-compile-rs/examples/bench.rs @@ -0,0 +1,86 @@ +//! cargo run --release --example bench: each fixture pipeline against its +//! compiled module (tests/expected/), µs per run. The pipeline gets a Payload +//! and gives a Payload; the module gets its struct and gives its struct. +#[path = "../tests/fixtures/pricing/mod.rs"] +mod pricing; +#[path = "../tests/expected/pricing.rs"] +mod pricing_compiled; +#[path = "../tests/fixtures/signup/mod.rs"] +mod signup; +#[path = "../tests/expected/signup.rs"] +mod signup_compiled; + +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use codeupipe::{Payload, Pipeline}; + +const RUNS: u32 = 20_000; + +// cup: allow-plain the timing loop of the bench: it measures the pipeline, so it is not a step of it +fn time(f: &mut impl FnMut()) -> Duration { + let started = Instant::now(); + for _ in 0..RUNS { + f(); + } + started.elapsed() +} + +/// The best of 9 rounds, the two alternated in each round so that the load of the machine falls on both. +// cup: allow-plain the timing loop of the bench: it measures the pipeline, so it is not a step of it +fn race(mut a: impl FnMut(), mut b: impl FnMut()) -> (f64, f64) { + for _ in 0..3 { + time(&mut a); + time(&mut b); + } + let (mut best_a, mut best_b) = (Duration::MAX, Duration::MAX); + for _ in 0..9 { + best_a = best_a.min(time(&mut a)); + best_b = best_b.min(time(&mut b)); + } + let us = |d: Duration| d.as_secs_f64() / f64::from(RUNS) * 1e6; + (us(best_a), us(best_b)) +} + +fn main() { + let pipeline = Pipeline::named(pricing::pricing_pipeline::PricingPipeline); + let mut payload = Payload::new(); + payload.insert("price", 100.0).insert("qty", 12.0); + let data = pricing_compiled::PricingData { + price: Some(100.0), + qty: Some(12.0), + bulk: None, + }; + let (dynamic, compiled) = race( + || { + black_box(pipeline.run(black_box(&payload)).ok()); + }, + || { + black_box(pricing_compiled::run(black_box(&data)).ok()); + }, + ); + println!( + "pricing (3 steps) Pipeline::run {dynamic:.3} µs compiled {compiled:.4} µs {:.0}x", + dynamic / compiled + ); + + let pipeline = Pipeline::named(signup::signup_pipeline::SignupPipeline); + let mut payload = Payload::new(); + payload + .insert("email", " Ada@Example.COM ") + .insert("coupon", "SAVE5"); + let data = signup_compiled::SignupData { + email: Some(" Ada@Example.COM ".into()), + coupon: Some("SAVE5".into()), + ..Default::default() + }; + let (dynamic, compiled) = race( + || { + black_box(pipeline.run(black_box(&payload)).ok()); + }, + || { + black_box(signup_compiled::run(black_box(&data)).ok()); + }, + ); + println!("signup (4 steps, text) Pipeline::run {dynamic:.3} µs compiled {compiled:.4} µs {:.1}x", dynamic / compiled); +} diff --git a/packages/codeupipe-compile-rs/src/compile_pipeline.rs b/packages/codeupipe-compile-rs/src/compile_pipeline.rs new file mode 100644 index 0000000..924a87a --- /dev/null +++ b/packages/codeupipe-compile-rs/src/compile_pipeline.rs @@ -0,0 +1,21 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use crate::compile_run::CompileRun; +use crate::emit_module_filter::EmitModuleFilter; +use crate::lower_steps_filter::LowerStepsFilter; +use crate::read_sources_filter::ReadSourcesFilter; +use crate::resolve_pipeline_filter::ResolvePipelineFilter; +use crate::type_keys_filter::TypeKeysFilter; + +/// The compiler, as a codeupipe pipeline: `Pipeline::named(CompilePipeline)`. +pub struct CompilePipeline; + +impl NamedPipeline for CompilePipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(ReadSourcesFilter); // the paths to syntax trees + pipeline.add_filter(ResolvePipelineFilter); // the pipeline to a list of steps, with their settings + pipeline.add_filter(LowerStepsFilter); // each step to a plain function on the struct + pipeline.add_filter(TypeKeysFilter); // each key to a field with one type + pipeline.add_filter(EmitModuleFilter); // the struct and the functions to one module + } +} diff --git a/packages/codeupipe-compile-rs/src/compile_run.rs b/packages/codeupipe-compile-rs/src/compile_run.rs new file mode 100644 index 0000000..691035a --- /dev/null +++ b/packages/codeupipe-compile-rs/src/compile_run.rs @@ -0,0 +1,35 @@ +use codeupipe::PipelineData; + +use crate::key_type::KeyType; +use crate::key_use::KeyUse; +use crate::lowered_step::LoweredStep; +use crate::source::Source; +use crate::step::StepPlan; + +/// What the compile pipeline carries from step to step: the input, and what +/// each step adds. +#[derive(Clone, Default)] +pub struct CompileRun { + /// Files or directories with the pipeline and its steps. + pub paths: Vec, + /// The name of the type that implements `NamedPipeline`. + pub pipeline: String, + /// The parsed files (ReadSourcesFilter). + pub sources: Vec, + /// The name of the compiled struct: `PricingPipeline` gives `PricingData` (ResolvePipelineFilter). + pub data: String, + /// Each step, in the order of the pipeline (ResolvePipelineFilter). + pub steps: Vec, + /// Each step as a plain function (LowerStepsFilter). + pub functions: Vec, + /// Each use of a key, in order (LowerStepsFilter). + pub uses: Vec, + /// The `use` items of the step files that the functions need (LowerStepsFilter). + pub imports: Vec, + /// Each key with its field and its type, in the order of first use (TypeKeysFilter). + pub keys: Vec<(String, String, KeyType)>, + /// The compiled module (EmitModuleFilter). + pub module: String, +} + +impl PipelineData for CompileRun {} diff --git a/packages/codeupipe-compile-rs/src/emit_module_filter.rs b/packages/codeupipe-compile-rs/src/emit_module_filter.rs new file mode 100644 index 0000000..38ff583 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/emit_module_filter.rs @@ -0,0 +1,81 @@ +use codeupipe::{Filter, StepError}; +use quote::{format_ident, quote}; + +use crate::compile_run::CompileRun; +use crate::step::{Role, StepPlan}; + +/// The compiled module: the struct of the keys, the function of each step, +/// and `run`, formatted as Rust. +pub struct EmitModuleFilter; + +impl Filter for EmitModuleFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + let data = format_ident!("{}", run.data); + let pipeline = &run.pipeline; + let head = [ + format!( + " Written by codeupipe-compile from {pipeline}. Do not edit it: compile again." + ), + String::new(), + " Plain Rust, with no codeupipe: a struct with one field for each key that".to_string(), + " the steps use, and one function for each step. It has no State and no".to_string(), + " Hooks. A Tap failure does not stop the run.".to_string(), + ]; + let struct_doc = + format!(" The data of {pipeline}: one field for each key that its steps use."); + let run_doc = format!(" Runs the steps of {pipeline}, in order, on a copy of `input`."); + let fields = run.keys.iter().map(|(key, field, key_type)| { + let field = format_ident!("{}", field); + let ty = key_type.rust(); + let doc = format!(" The key {key:?}."); + quote!(#[doc = #doc] pub #field: Option<#ty>,) + }); + let functions = run.functions.iter().map(|lowered| &lowered.item); + let mut lowered = run.functions.iter(); + let mut calls = Vec::new(); + for plan in &run.steps { + let mut next = + || format_ident!("{}", lowered.next().expect("a function for each step").name); + calls.push(match plan { + StepPlan::Filter(_) => { + let f = next(); + quote!(#f(&mut payload)?;) + } + StepPlan::Valve(..) => { + let (predicate, filter) = (next(), next()); + quote!(if #predicate(&payload) { #filter(&mut payload)?; }) + } + StepPlan::Tap(_) => { + let t = next(); + quote!(let _ = #t(&payload);) + } + }); + } + debug_assert!(run + .functions + .iter() + .all(|f| matches!(f.role, Role::Filter | Role::Predicate | Role::Tap))); + let imports = &run.imports; + let file = quote! { + #(#![doc = #head])* + #(#imports)* + #[doc = " The error of a step."] + pub type StepError = Box; + #[doc = #struct_doc] + #[derive(Clone, Debug, Default, PartialEq)] + pub struct #data { + #(#fields)* + } + #(#functions)* + #[doc = #run_doc] + pub fn run(input: &#data) -> Result<#data, StepError> { + let mut payload = input.clone(); + #(#calls)* + Ok(payload) + } + }; + let tree: syn::File = syn::parse2(file)?; + run.module = prettyplease::unparse(&tree); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/src/key_type.rs b/packages/codeupipe-compile-rs/src/key_type.rs new file mode 100644 index 0000000..e744283 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/key_type.rs @@ -0,0 +1,31 @@ +use proc_macro2::TokenStream; +use quote::quote; + +/// The type of a key in the compiled struct: a number (`f64`), true or +/// false (`bool`), or text (`String`). Each field is an `Option`, because a +/// Payload can lack a key. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KeyType { + Number, + Bool, + Text, +} + +// cup: allow-plain a value of the compiler: its name in a message, and its type in Rust +impl KeyType { + pub fn name(self) -> &'static str { + match self { + KeyType::Number => "a number", + KeyType::Bool => "true or false", + KeyType::Text => "text", + } + } + + pub fn rust(self) -> TokenStream { + match self { + KeyType::Number => quote!(f64), + KeyType::Bool => quote!(bool), + KeyType::Text => quote!(String), + } + } +} diff --git a/packages/codeupipe-compile-rs/src/key_use.rs b/packages/codeupipe-compile-rs/src/key_use.rs new file mode 100644 index 0000000..9103e7c --- /dev/null +++ b/packages/codeupipe-compile-rs/src/key_use.rs @@ -0,0 +1,15 @@ +use crate::key_type::KeyType; +use crate::place::Place; + +/// How a step uses a key: a read or a write with a type, or a use with no +/// type (`has`, `remove`, a write of an expression). +#[derive(Clone, Debug)] +pub struct KeyUse { + pub key: String, + /// The type that this use gives the key, when it gives one. + pub key_type: Option, + /// A write of a value whose type the compiler cannot tell. + pub untyped_write: bool, + pub step: String, + pub place: Place, +} diff --git a/packages/codeupipe-compile-rs/src/lib.rs b/packages/codeupipe-compile-rs/src/lib.rs new file mode 100644 index 0000000..ac9746f --- /dev/null +++ b/packages/codeupipe-compile-rs/src/lib.rs @@ -0,0 +1,52 @@ +//! codeupipe-compile: writes a codeupipe pipeline in Rust as plain Rust: a +//! struct with one field for each key, and one function for each step, with +//! no codeupipe. Code that compiles gives the output of the pipeline. A +//! pipeline for which the compiler cannot keep that rule is refused, with +//! the place. +mod compile_pipeline; +mod compile_run; +mod emit_module_filter; +mod key_type; +mod key_use; +mod lower_steps_filter; +mod lowered_step; +mod payload_rewriter; +mod place; +mod read_sources_filter; +mod refusal; +mod resolve_pipeline_filter; +mod source; +mod step; +mod type_keys_filter; + +pub use compile_pipeline::CompilePipeline; +pub use compile_run::CompileRun; +pub use place::Place; +pub use refusal::Refusal; + +use codeupipe::Pipeline; + +/// Compiles the pipeline `pipeline` (the name of its type) from the Rust files under `paths`, to the text of a module. +// cup: allow-plain the entry of the library: it runs the compile pipeline and gives the module or the refusal +pub fn compile(paths: &[String], pipeline: &str) -> Result { + let run = CompileRun { + paths: paths.to_vec(), + pipeline: pipeline.to_string(), + ..CompileRun::default() + }; + match Pipeline::named(CompilePipeline).run(&run) { + Ok(done) => Ok(done.module), + Err(error) => match error.downcast::() { + Ok(refusal) => Err(*refusal), + Err(other) => Err(Refusal::new( + Place { + path: paths.first().cloned().unwrap_or_default(), + line: 0, + col: 0, + }, + format!("The compiler could not run: {other}"), + "Check that the files can be read", + )), + }, + } +} diff --git a/packages/codeupipe-compile-rs/src/lower_steps_filter.rs b/packages/codeupipe-compile-rs/src/lower_steps_filter.rs new file mode 100644 index 0000000..34f77e7 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/lower_steps_filter.rs @@ -0,0 +1,184 @@ +use codeupipe::{Filter, StepError}; +use quote::{format_ident, ToTokens}; +use syn::spanned::Spanned; +use syn::visit_mut::VisitMut; +use syn::{FnArg, ImplItem, ImplItemFn, Item, Pat, Type, UseTree}; + +use crate::compile_run::CompileRun; +use crate::lowered_step::LoweredStep; +use crate::payload_rewriter::PayloadRewriter; +use crate::refusal::Refusal; +use crate::source::Source; +use crate::step::{Role, Step, StepPlan}; + +/// Each step to a plain function on the compiled struct: the body of its +/// `call` (a Filter), `test` (a Predicate) or `observe` (a Tap), rewritten by +/// PayloadRewriter. The `use` items of the step file come with it. +pub struct LowerStepsFilter; + +impl LowerStepsFilter { + fn snake(name: &str) -> String { + let mut out = String::new(); + for (at, c) in name.chars().enumerate() { + if c.is_uppercase() && at > 0 { + out.push('_'); + } + out.extend(c.to_lowercase()); + } + out + } + + fn type_name(ty: &Type) -> String { + match ty { + Type::Path(path) => path + .path + .segments + .last() + .map(|s| s.ident.to_string()) + .unwrap_or_default(), + _ => String::new(), + } + } + + /// A `use` that the compiled module keeps: not of codeupipe, and not of the project (`super`, `crate`, `self`). + fn kept_use(tree: &UseTree) -> bool { + match tree { + UseTree::Path(path) => { + !["codeupipe", "super", "crate", "self"].contains(&path.ident.to_string().as_str()) + } + UseTree::Name(name) => name.ident != "codeupipe", + _ => true, + } + } + + /// The method of the role in `impl for `, with its file. + fn method<'a>(sources: &'a [Source], step: &Step) -> Option<(&'a Source, &'a ImplItemFn)> { + let (trait_name, method) = match step.role { + Role::Filter => ("Filter", "call"), + Role::Predicate => ("Predicate", "test"), + Role::Tap => ("Tap", "observe"), + }; + sources.iter().find_map(|source| { + source.tree.items.iter().find_map(|item| match item { + Item::Impl(imp) + if Self::type_name(&imp.self_ty) == step.type_name + && imp.trait_.as_ref().is_some_and(|(_, path, _)| { + path.segments.last().is_some_and(|s| s.ident == trait_name) + }) => + { + imp.items.iter().find_map(|item| match item { + ImplItem::Fn(func) if func.sig.ident == method => Some((source, func)), + _ => None, + }) + } + _ => None, + }) + }) + } + + fn lower(run: &mut CompileRun, step: &Step, name: String) -> Result<(), Refusal> { + let sources = run.sources.clone(); + let Some((source, method)) = Self::method(&sources, step) else { + let role = format!("{:?}", step.role); + return Err(Refusal::new( + step.place.clone(), + format!( + "There is no `impl {role} for {}` in the paths", + step.type_name + ), + "Give the file of each step", + )); + }; + let payload = method.sig.inputs.iter().find_map(|input| match input { + FnArg::Typed(typed) => match &*typed.pat { + Pat::Ident(ident) => Some(ident.ident.to_string()), + _ => None, + }, + FnArg::Receiver(_) => None, + }); + let Some(payload) = payload else { + return Err(Refusal::new( + crate::place::Place::of(&source.path, method.sig.span()), + "The Payload parameter has no name", + "Give it a name", + )); + }; + let mut rewriter = PayloadRewriter { + path: &source.path, + step: &step.type_name, + payload: payload.clone(), + settings: &step.settings, + writes: step.role == Role::Filter, + uses: Vec::new(), + refusal: None, + }; + let mut block = method.block.clone(); + rewriter.visit_block_mut(&mut block); + rewriter.check_left(block.to_token_stream()); + if let Some(refusal) = rewriter.refusal { + return Err(refusal); + } + run.uses.extend(rewriter.uses); + let data = format_ident!("{}", run.data); + let payload = format_ident!("{}", payload); + let function = format_ident!("{}", name); + let doc = format!(" {}", step.type_name); + let item: syn::ItemFn = match step.role { + Role::Filter => { + syn::parse_quote!(#[doc = #doc] fn #function(#payload: &mut #data) -> Result<(), StepError> #block) + } + Role::Predicate => { + syn::parse_quote!(#[doc = #doc] fn #function(#payload: &#data) -> bool #block) + } + Role::Tap => { + syn::parse_quote!(#[doc = #doc] fn #function(#payload: &#data) -> Result<(), StepError> #block) + } + }; + for item in &source.tree.items { + if let Item::Use(import) = item { + if Self::kept_use(&import.tree) + && !run.imports.iter().any(|kept| { + kept.to_token_stream().to_string() == import.to_token_stream().to_string() + }) + { + run.imports.push(import.clone()); + } + } + } + run.functions.push(LoweredStep { + role: step.role, + name, + type_name: step.type_name.clone(), + item, + }); + Ok(()) + } +} + +impl Filter for LowerStepsFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + for (at, plan) in run.steps.clone().iter().enumerate() { + let number = format!("cup_{:02}", at + 1); + match plan { + StepPlan::Filter(step) | StepPlan::Tap(step) => Self::lower( + run, + step, + format!("{number}_{}", Self::snake(&step.type_name)), + )?, + StepPlan::Valve(predicate, filter) => { + Self::lower( + run, + predicate, + format!("{number}_when_{}", Self::snake(&predicate.type_name)), + )?; + Self::lower( + run, + filter, + format!("{number}_{}", Self::snake(&filter.type_name)), + )?; + } + } + } + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/src/lowered_step.rs b/packages/codeupipe-compile-rs/src/lowered_step.rs new file mode 100644 index 0000000..599d893 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/lowered_step.rs @@ -0,0 +1,10 @@ +use crate::step::Role; + +/// A step as a plain function of the compiled module. +#[derive(Clone)] +pub struct LoweredStep { + pub role: Role, + pub name: String, + pub type_name: String, + pub item: syn::ItemFn, +} diff --git a/packages/codeupipe-compile-rs/src/main.rs b/packages/codeupipe-compile-rs/src/main.rs new file mode 100644 index 0000000..b867033 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/main.rs @@ -0,0 +1,53 @@ +//! codeupipe-compile PATH... PIPELINE [--out FILE] +//! +//! Writes the pipeline PIPELINE (the name of its type) from the Rust files +//! under the paths as plain Rust, to FILE or to stdout. Exit code: 0 the +//! module is written, 1 the pipeline is refused (the message names the place +//! and the fix), 2 the command could not run. Nothing is written on 1 or 2. +use std::process::ExitCode; + +use codeupipe_compile::compile; + +// cup: allow-plain the entry of the program: it reads the arguments and writes the module +fn main() -> ExitCode { + let mut args: Vec = std::env::args().skip(1).collect(); + let out = match args.iter().position(|arg| arg == "--out") { + Some(at) if at + 1 < args.len() => { + let file = args.remove(at + 1); + args.remove(at); + Some(file) + } + Some(_) => return usage(), + None => None, + }; + let Some(pipeline) = args.pop() else { + return usage(); + }; + if args.is_empty() { + return usage(); + } + match compile(&args, &pipeline) { + Ok(module) => match out { + Some(file) => match std::fs::write(&file, module) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("{file}: {error}"); + ExitCode::from(2) + } + }, + None => { + print!("{module}"); + ExitCode::SUCCESS + } + }, + Err(refusal) => { + eprintln!("{refusal}"); + ExitCode::from(1) + } + } +} + +fn usage() -> ExitCode { + eprintln!("Usage: codeupipe-compile PATH... PIPELINE [--out FILE]"); + ExitCode::from(2) +} diff --git a/packages/codeupipe-compile-rs/src/payload_rewriter.rs b/packages/codeupipe-compile-rs/src/payload_rewriter.rs new file mode 100644 index 0000000..ee7ba2f --- /dev/null +++ b/packages/codeupipe-compile-rs/src/payload_rewriter.rs @@ -0,0 +1,348 @@ +use proc_macro2::{TokenStream, TokenTree}; +use quote::{format_ident, ToTokens}; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned; +use syn::visit_mut::{self, VisitMut}; +use syn::{Expr, Lit, Member, Token}; + +use crate::key_type::KeyType; +use crate::key_use::KeyUse; +use crate::place::Place; +use crate::refusal::Refusal; + +const USE_ONLY: &str = + "Use the Payload only through get(..).and_then(Value::as_...), has, insert and remove"; + +/// Rewrites the body of a step for the compiled struct: a key of the Payload +/// becomes a field, and `self.setting` becomes the literal that the pipeline +/// gave it. Each other use of the Payload, of `self` or of `Value` is refused. +pub struct PayloadRewriter<'a> { + pub path: &'a str, + pub step: &'a str, + /// The name of the Payload parameter of the step. + pub payload: String, + pub settings: &'a [(String, Expr)], + pub writes: bool, + pub uses: Vec, + pub refusal: Option, +} + +// cup: allow-plain the rewrite of a step body: a walk of its syntax tree, called by LowerStepsFilter +impl PayloadRewriter<'_> { + /// The field of the compiled struct for a key: the key when it is a name in + /// Rust, or the key with each other character made `_`. + pub fn field_of(key: &str) -> String { + let mut field: String = key + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' { + c + } else { + '_' + } + }) + .collect(); + if field.is_empty() + || field.starts_with(|c: char| c.is_ascii_digit()) + || syn::parse_str::(&field).is_err() + { + field = format!("key_{field}"); + } + field + } + + /// The type of a value that a step writes, when the code shows it. + fn written_type(expr: &Expr) -> Option { + match expr { + Expr::Lit(lit) => match lit.lit { + Lit::Float(_) | Lit::Int(_) => Some(KeyType::Number), + Lit::Bool(_) => Some(KeyType::Bool), + Lit::Str(_) => Some(KeyType::Text), + _ => None, + }, + Expr::Unary(unary) => { + Self::written_type(&unary.expr).filter(|ty| *ty == KeyType::Number) + } + Expr::Paren(paren) => Self::written_type(&paren.expr), + Expr::Macro(mac) if mac.mac.path.is_ident("format") => Some(KeyType::Text), + Expr::Call(call) if matches!(&*call.func, Expr::Path(p) if p.path.segments.len() == 2 && p.path.segments[0].ident == "String") => { + Some(KeyType::Text) + } + Expr::MethodCall(call) + if ["to_string", "to_owned", "to_lowercase", "to_uppercase"] + .contains(&call.method.to_string().as_str()) => + { + Some(KeyType::Text) + } + _ => None, + } + } + + /// The type that `Value::as_f64`, `Value::as_bool` or `Value::as_str` reads, from `.and_then(..)`. + fn read_type(arg: &Expr) -> Option { + let name = match arg { + Expr::Path(path) + if path.path.segments.len() == 2 && path.path.segments[0].ident == "Value" => + { + path.path.segments[1].ident.to_string() + } + Expr::Closure(closure) if closure.inputs.len() == 1 => match &*closure.body { + Expr::MethodCall(call) if call.args.is_empty() => call.method.to_string(), + _ => return None, + }, + _ => return None, + }; + match name.as_str() { + "as_f64" => Some(KeyType::Number), + "as_bool" => Some(KeyType::Bool), + "as_str" => Some(KeyType::Text), + _ => None, + } + } + + fn is_self(expr: &Expr) -> bool { + matches!(expr, Expr::Path(path) if path.path.is_ident("self")) + } + + fn refuse(&mut self, span: proc_macro2::Span, what: impl Into, fix: &str) { + if self.refusal.is_none() { + self.refusal = Some(Refusal::new(Place::of(self.path, span), what, fix)); + } + } + + fn is_payload(&self, expr: &Expr) -> bool { + matches!(expr, Expr::Path(path) if path.path.is_ident(&self.payload)) + } + + fn record( + &mut self, + key: &str, + key_type: Option, + untyped_write: bool, + span: proc_macro2::Span, + ) { + self.uses.push(KeyUse { + key: key.to_string(), + key_type, + untyped_write, + step: self.step.to_string(), + place: Place::of(self.path, span), + }); + } + + /// The text of a key: a literal, perhaps behind `&`, `.to_string()`, `String::from(..)`. Settings are put in first. + fn key_of(&mut self, arg: &Expr) -> Option { + let mut arg = arg.clone(); + self.visit_expr_mut(&mut arg); + let mut at = &arg; + loop { + match at { + Expr::Lit(lit) => { + if let Lit::Str(text) = &lit.lit { + return Some(text.value()); + } + break; + } + Expr::Reference(reference) => at = &reference.expr, + Expr::Paren(paren) => at = &paren.expr, + Expr::MethodCall(call) + if call.args.is_empty() + && ["to_string", "as_str", "as_ref", "to_owned", "into", "clone"] + .contains(&call.method.to_string().as_str()) => + { + at = &call.receiver + } + Expr::Call(call) + if call.args.len() == 1 + && matches!(&*call.func, Expr::Path(p) if p.path.segments.len() == 2 && p.path.segments[0].ident == "String") => + { + at = &call.args[0] + } + _ => break, + } + } + self.refuse( + arg.span(), + "A key that is not text in the code", + "Write the key as text in the step, or as a setting that the pipeline gives as text", + ); + None + } + + /// `payload.get(K).and_then(Value::as_x)` to the field, or None. + fn typed_read(&mut self, expr: &Expr) -> Option { + let Expr::MethodCall(outer) = expr else { + return None; + }; + if outer.method != "and_then" || outer.args.len() != 1 { + return None; + } + let Expr::MethodCall(get) = &*outer.receiver else { + return None; + }; + if get.method != "get" || get.args.len() != 1 || !self.is_payload(&get.receiver) { + return None; + } + let key_type = Self::read_type(&outer.args[0])?; + let key = self.key_of(&get.args[0])?; + self.record(&key, Some(key_type), false, get.span()); + let payload = format_ident!("{}", self.payload); + let field = format_ident!("{}", Self::field_of(&key)); + Some(match key_type { + KeyType::Text => syn::parse_quote!(#payload.#field.as_deref()), + _ => syn::parse_quote!(#payload.#field), + }) + } + + /// A method of the Payload: has, insert or remove. `expr` is the call. + fn payload_call(&mut self, expr: &mut Expr) { + let Expr::MethodCall(call) = expr else { return }; + let method = call.method.to_string(); + let span = call.span(); + let payload = format_ident!("{}", self.payload); + match (method.as_str(), call.args.len()) { + ("has", 1) => { + let Some(key) = self.key_of(&call.args[0]) else { return }; + self.record(&key, None, false, span); + let field = format_ident!("{}", Self::field_of(&key)); + *expr = syn::parse_quote!(#payload.#field.is_some()); + } + ("insert" | "remove", _) if !self.writes => { + self.refuse(span, format!("`{}.{method}(..)` in a step that only reads", self.payload), "Only a Filter writes"); + } + ("insert", 2) => { + let Some(key) = self.key_of(&call.args[0]) else { return }; + let mut value = call.args[1].clone(); + self.visit_expr_mut(&mut value); + let key_type = Self::written_type(&value); + self.record(&key, key_type, key_type.is_none(), span); + let field = format_ident!("{}", Self::field_of(&key)); + *expr = syn::parse_quote!(#payload.#field = Some(::core::convert::Into::into(#value))); + } + ("remove", 1) => { + let Some(key) = self.key_of(&call.args[0]) else { return }; + self.record(&key, None, false, span); + let field = format_ident!("{}", Self::field_of(&key)); + *expr = syn::parse_quote!(#payload.#field = None); + } + ("get", _) => self.refuse(span, format!("A read with no type: `{}.get(..)`", self.payload), "Read a key with .and_then(Value::as_f64), .and_then(Value::as_bool) or .and_then(Value::as_str)"), + _ => { + let shown = call.to_token_stream().to_string().replace(" . ", ".").replace(" (", "(").replace("( ", "(").replace(" )", ")"); + self.refuse(span, format!("`{shown}` is not in plain data"), USE_ONLY); + } + } + } + + /// The names that must not be left in the compiled function: the Payload, `self` and `Value`. + pub fn check_left(&mut self, tokens: TokenStream) { + for tree in tokens { + match tree { + TokenTree::Ident(ident) if ident == self.payload.as_str() => {} + TokenTree::Ident(ident) if ident == "self" => self.refuse( + ident.span(), + "`self` used in another way", + "Use self only to read a setting: self.field", + ), + TokenTree::Ident(ident) if ident == "Value" || ident == "Payload" => self.refuse( + ident.span(), + format!("`{ident}` used in another way"), + USE_ONLY, + ), + TokenTree::Group(group) => self.check_left(group.stream()), + _ => {} + } + } + } +} + +// cup: allow-plain the rewrite of a step body: a walk of its syntax tree, called by LowerStepsFilter +impl VisitMut for PayloadRewriter<'_> { + fn visit_expr_mut(&mut self, expr: &mut Expr) { + if self.refusal.is_some() { + return; + } + if let Some(read) = self.typed_read(expr) { + *expr = read; + return; + } + match expr { + Expr::Field(field) if Self::is_self(&field.base) => { + let Member::Named(name) = &field.member else { + return; + }; + match self.settings.iter().find(|(setting, _)| name == setting) { + Some((_, value)) => *expr = value.clone(), + None => { + let span = field.span(); + self.refuse( + span, + format!("`self.{name}` is not a setting that the pipeline gives"), + "Give each setting in the call that makes the step", + ); + } + } + return; + } + Expr::MethodCall(call) if Self::is_self(&call.receiver) => { + let span = call.span(); + let method = call.method.to_string(); + self.refuse( + span, + format!("A step that calls a method of its own: `self.{method}()`"), + "Put the code of the step in its method", + ); + return; + } + Expr::MethodCall(call) if matches!(&*call.receiver, Expr::MethodCall(inner) if self.is_payload(&inner.receiver) && (inner.method == "insert" || inner.method == "remove")) => + { + let span = call.span(); + self.refuse( + span, + "A chain of writes on the Payload", + "Write one key in each statement", + ); + return; + } + Expr::MethodCall(call) if self.is_payload(&call.receiver) => { + self.payload_call(expr); + return; + } + Expr::Path(path) if path.path.is_ident(&self.payload) => { + let span = path.span(); + self.refuse(span, "The Payload given to other code", USE_ONLY); + return; + } + _ => {} + } + visit_mut::visit_expr_mut(self, expr); + } + + /// The arguments of a macro such as format! or println! are code too: they are read and rewritten. + fn visit_macro_mut(&mut self, mac: &mut syn::Macro) { + if self.refusal.is_some() { + return; + } + match mac.parse_body_with(Punctuated::::parse_terminated) { + Ok(mut args) => { + for arg in args.iter_mut() { + self.visit_expr_mut(arg); + } + mac.tokens = args.to_token_stream(); + } + Err(_) => { + let names = mac.tokens.to_string(); + if names + .split(|c: char| !c.is_alphanumeric() && c != '_') + .any(|word| word == self.payload || word == "self") + { + let span = mac.span(); + self.refuse( + span, + "A macro whose code the compiler cannot read uses the Payload or self", + "Read the Payload into a name first, and give the name to the macro", + ); + } + } + } + } +} diff --git a/packages/codeupipe-compile-rs/src/place.rs b/packages/codeupipe-compile-rs/src/place.rs new file mode 100644 index 0000000..ba0305c --- /dev/null +++ b/packages/codeupipe-compile-rs/src/place.rs @@ -0,0 +1,20 @@ +use proc_macro2::Span; + +/// Where a refusal points: a file, a line and a column, from 1. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Place { + pub path: String, + pub line: usize, + pub col: usize, +} + +impl Place { + pub fn of(path: &str, span: Span) -> Place { + let start = span.start(); + Place { + path: path.to_string(), + line: start.line, + col: start.column + 1, + } + } +} diff --git a/packages/codeupipe-compile-rs/src/read_sources_filter.rs b/packages/codeupipe-compile-rs/src/read_sources_filter.rs new file mode 100644 index 0000000..c4e0fcf --- /dev/null +++ b/packages/codeupipe-compile-rs/src/read_sources_filter.rs @@ -0,0 +1,63 @@ +use std::fs; +use std::path::Path; + +use codeupipe::{Filter, StepError}; + +use crate::compile_run::CompileRun; +use crate::place::Place; +use crate::refusal::Refusal; +use crate::source::Source; + +/// Reads and parses each Rust file under the paths, in the order of their names. +pub struct ReadSourcesFilter; + +impl ReadSourcesFilter { + fn files(path: &Path, into: &mut Vec) -> Result<(), StepError> { + if path.is_dir() { + let mut entries: Vec<_> = fs::read_dir(path)? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .collect(); + entries.sort(); + for entry in entries { + Self::files(&entry, into)?; + } + } else if path.extension().is_some_and(|ext| ext == "rs") { + into.push(path.to_string_lossy().into_owned()); + } + Ok(()) + } +} + +impl Filter for ReadSourcesFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + let mut files = Vec::new(); + for path in &run.paths { + if !Path::new(path).exists() { + return Err(Refusal::new( + Place { + path: path.clone(), + line: 0, + col: 0, + }, + "There is no such file or directory", + "Give the files of the pipeline and its steps", + ) + .into()); + } + Self::files(Path::new(path), &mut files)?; + } + for path in files { + let text = fs::read_to_string(&path)?; + let tree = syn::parse_file(&text).map_err(|error| { + Refusal::new( + Place::of(&path, error.span()), + format!("The file does not parse: {error}"), + "Fix the syntax", + ) + })?; + run.sources.push(Source { path, tree }); + } + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/src/refusal.rs b/packages/codeupipe-compile-rs/src/refusal.rs new file mode 100644 index 0000000..8e1ae96 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/refusal.rs @@ -0,0 +1,32 @@ +use std::fmt; + +use crate::place::Place; + +/// A pipeline that the compiler does not compile, with the place and the +/// fix. Code that compiles gives the output of the pipeline; a shape for +/// which the compiler cannot keep that rule is refused. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Refusal { + pub place: Place, + pub what: String, + pub fix: String, +} + +impl Refusal { + pub fn new(place: Place, what: impl Into, fix: impl Into) -> Refusal { + Refusal { + place, + what: what.into(), + fix: fix.into(), + } + } +} + +impl fmt::Display for Refusal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Place { path, line, col } = &self.place; + write!(f, "{path}:{line}:{col} {}. {}", self.what, self.fix) + } +} + +impl std::error::Error for Refusal {} diff --git a/packages/codeupipe-compile-rs/src/resolve_pipeline_filter.rs b/packages/codeupipe-compile-rs/src/resolve_pipeline_filter.rs new file mode 100644 index 0000000..3e40f5b --- /dev/null +++ b/packages/codeupipe-compile-rs/src/resolve_pipeline_filter.rs @@ -0,0 +1,390 @@ +use codeupipe::{Filter, StepError}; +use syn::spanned::Spanned; +use syn::{Expr, Fields, ImplItem, Item, Lit, Member, Stmt, Type, UnOp}; + +use crate::compile_run::CompileRun; +use crate::place::Place; +use crate::refusal::Refusal; +use crate::source::Source; +use crate::step::{Role, Step, StepPlan}; + +/// Finds `impl NamedPipeline for `, and reads its `add_steps`: each +/// step, its role, and each setting with the literal that the pipeline +/// gives it (from a struct literal or a constructor such as `new`). +pub struct ResolvePipelineFilter; + +impl ResolvePipelineFilter { + /// The last name of a type: `Filter` for `codeupipe::Filter`. + fn last_name(path: &syn::Path) -> String { + path.segments + .last() + .map(|segment| segment.ident.to_string()) + .unwrap_or_default() + } + + fn type_name(ty: &Type) -> String { + match ty { + Type::Path(path) => Self::last_name(&path.path), + _ => String::new(), + } + } + + /// A literal setting: text, a number (also negative), or true or false. + fn is_literal(expr: &Expr) -> bool { + match expr { + Expr::Lit(_) => true, + Expr::Unary(unary) => { + matches!(unary.op, UnOp::Neg(_)) + && matches!(&*unary.expr, Expr::Lit(lit) if matches!(lit.lit, Lit::Int(_) | Lit::Float(_))) + } + Expr::Paren(paren) => Self::is_literal(&paren.expr), + _ => false, + } + } + + fn has_type(sources: &[Source], name: &str) -> bool { + sources.iter().any(|source| { + source.tree.items.iter().any(|item| match item { + Item::Struct(item) => item.ident == name, + Item::Enum(item) => item.ident == name, + _ => false, + }) + }) + } + + /// The fields of `struct name`, with their types, and the file. + fn fields(sources: &[Source], name: &str) -> Vec<(String, Type)> { + for source in sources { + for item in &source.tree.items { + if let Item::Struct(item) = item { + if item.ident == name { + if let Fields::Named(named) = &item.fields { + return named + .named + .iter() + .filter_map(|field| { + Some((field.ident.as_ref()?.to_string(), field.ty.clone())) + }) + .collect(); + } + } + } + } + } + Vec::new() + } + + /// The setting as the step keeps it: a `String` field gets `String::from(literal)`. + fn kept(literal: Expr, ty: Option<&Type>) -> Expr { + match ty { + Some(ty) if Self::type_name(ty) == "String" => { + syn::parse_quote!(String::from(#literal)) + } + _ => literal, + } + } + + /// `T`, `T { field: literal }`, or `T::new(literal, ...)` to a step. + fn step(sources: &[Source], path: &str, expr: &Expr, role: Role) -> Result { + let place = Place::of(path, expr.span()); + let fix = "Make the step as T, as T { field: literal }, or as T::new(literal, ...)"; + let (type_name, settings) = match expr { + Expr::Path(p) if p.path.segments.len() == 1 => (Self::last_name(&p.path), Vec::new()), + Expr::Struct(literal) => { + let name = Self::last_name(&literal.path); + let types = Self::fields(sources, &name); + let mut settings = Vec::new(); + for field in &literal.fields { + if !Self::is_literal(&field.expr) { + return Err(Refusal::new( + Place::of(path, field.expr.span()), + "A setting that is not a literal", + "Give each setting as text, a number, or true or false", + )); + } + let Member::Named(ident) = &field.member else { + return Err(Refusal::new(place, "A step of a tuple struct", fix)); + }; + let ty = types + .iter() + .find(|(name, _)| ident == name) + .map(|(_, ty)| ty); + settings.push((ident.to_string(), Self::kept(field.expr.clone(), ty))); + } + (name, settings) + } + Expr::Call(call) => { + let Expr::Path(func) = &*call.func else { + return Err(Refusal::new( + place, + "A step that is made in another way", + fix, + )); + }; + if func.path.segments.len() != 2 { + return Err(Refusal::new( + place, + "A step that is made in another way", + fix, + )); + } + let name = func.path.segments[0].ident.to_string(); + let constructor = func.path.segments[1].ident.to_string(); + for arg in &call.args { + if !Self::is_literal(arg) { + return Err(Refusal::new( + Place::of(path, arg.span()), + "A setting that is not a literal", + "Give each setting as text, a number, or true or false", + )); + } + } + let args: Vec = call.args.iter().cloned().collect(); + ( + name.clone(), + Self::constructor(sources, &name, &constructor, &args, &place)?, + ) + } + _ => { + return Err(Refusal::new( + place, + "A step that is made in another way", + fix, + )) + } + }; + if !Self::has_type(sources, &type_name) { + return Err(Refusal::new( + place, + format!("There is no source for `{type_name}` in the paths"), + "Give the file of each step", + )); + } + Ok(Step { + role, + type_name, + settings, + place, + }) + } + + /// The settings that `T::constructor(args)` gives: each field must be a parameter (perhaps made into a + /// `String`) or a literal. + fn constructor( + sources: &[Source], + name: &str, + constructor: &str, + args: &[Expr], + place: &Place, + ) -> Result, Refusal> { + for source in sources { + for item in &source.tree.items { + let Item::Impl(imp) = item else { continue }; + if imp.trait_.is_some() || Self::type_name(&imp.self_ty) != name { + continue; + } + for impl_item in &imp.items { + let ImplItem::Fn(func) = impl_item else { + continue; + }; + if func.sig.ident != constructor { + continue; + } + let params: Vec = func + .sig + .inputs + .iter() + .filter_map(|input| match input { + syn::FnArg::Typed(typed) => match &*typed.pat { + syn::Pat::Ident(ident) => Some(ident.ident.to_string()), + _ => None, + }, + _ => None, + }) + .collect(); + let computes = |span| { + Refusal::new( + Place::of(&source.path, span), + format!("`{name}::{constructor}` computes its settings"), + "Keep each parameter as it is, and compute in the method of the step", + ) + }; + let Some(Stmt::Expr(Expr::Struct(made), None)) = func.block.stmts.last() else { + return Err(computes(func.block.span())); + }; + if func.block.stmts.len() != 1 { + return Err(computes(func.block.span())); + } + let types = Self::fields(sources, name); + let mut settings = Vec::new(); + for field in &made.fields { + let Member::Named(ident) = &field.member else { + return Err(computes(field.span())); + }; + let value = Self::parameter(&field.expr) + .and_then(|param| params.iter().position(|p| *p == param)) + .map(|at| args[at].clone()); + let value = match value { + Some(value) => value, + None if Self::is_literal(&field.expr) => field.expr.clone(), + None => return Err(computes(field.expr.span())), + }; + let ty = types + .iter() + .find(|(field_name, _)| ident == field_name) + .map(|(_, ty)| ty); + settings.push((ident.to_string(), Self::kept(value, ty))); + } + return Ok(settings); + } + } + } + Err(Refusal::new( + place.clone(), + format!("There is no `{name}::{constructor}` in the paths"), + "Give the file of each step", + )) + } + + /// The parameter that a field keeps: `rate`, `key.to_string()`, `key.into()`, `String::from(key)`. + fn parameter(expr: &Expr) -> Option { + match expr { + Expr::Path(path) if path.path.segments.len() == 1 => Some(Self::last_name(&path.path)), + Expr::MethodCall(call) + if call.args.is_empty() + && ["to_string", "into", "to_owned"] + .contains(&call.method.to_string().as_str()) => + { + Self::parameter(&call.receiver) + } + Expr::Call(call) + if call.args.len() == 1 + && matches!(&*call.func, Expr::Path(p) if p.path.segments.len() == 2 && p.path.segments[0].ident == "String" && p.path.segments[1].ident == "from") => + { + Self::parameter(&call.args[0]) + } + _ => None, + } + } +} + +impl Filter for ResolvePipelineFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + let found = run.sources.iter().find_map(|source| { + source.tree.items.iter().find_map(|item| match item { + Item::Impl(imp) + if Self::type_name(&imp.self_ty) == run.pipeline + && imp.trait_.as_ref().is_some_and(|(_, path, _)| { + Self::last_name(path) == "NamedPipeline" + }) => + { + Some((source, imp)) + } + _ => None, + }) + }); + let Some((source, imp)) = found else { + let path = run.paths.first().cloned().unwrap_or_default(); + return Err(Refusal::new( + Place { + path, + line: 0, + col: 0, + }, + format!( + "There is no `impl NamedPipeline for {}` in the paths", + run.pipeline + ), + "Give the file of the pipeline, and its name", + ) + .into()); + }; + if let Some((_, path, _)) = &imp.trait_ { + if let syn::PathArguments::AngleBracketed(args) = + &path.segments.last().expect("a trait has a name").arguments + { + let payload = args.args.iter().all(|arg| matches!(arg, syn::GenericArgument::Type(ty) if Self::type_name(ty) == "Payload")); + if !payload { + return Err(Refusal::new( + Place::of(&source.path, path.span()), + "A pipeline of a type of your own", + "It is plain data already: compile a pipeline of Payload", + ) + .into()); + } + } + } + let name = run + .pipeline + .strip_suffix("Pipeline") + .unwrap_or(&run.pipeline); + run.data = format!("{}Data", if name.is_empty() { &run.pipeline } else { name }); + let add_steps = imp.items.iter().find_map(|item| match item { + ImplItem::Fn(func) if func.sig.ident == "add_steps" => Some(func), + _ => None, + }); + let Some(add_steps) = add_steps else { + return Err(Refusal::new( + Place::of(&source.path, imp.span()), + "The pipeline has no add_steps", + "Add its steps in add_steps", + ) + .into()); + }; + let mut steps = Vec::new(); + for stmt in &add_steps.block.stmts { + let Stmt::Expr(Expr::MethodCall(call), Some(_)) = stmt else { + if matches!(stmt, Stmt::Local(_)) { + continue; // a name for a setting: the step that uses it is refused, since a setting is a literal + } + return Err(Refusal::new( + Place::of(&source.path, stmt.span()), + "add_steps does more than add steps", + "Keep only the add_... calls", + ) + .into()); + }; + let place = Place::of(&source.path, call.span()); + let args: Vec<&Expr> = call.args.iter().collect(); + let plan = match (call.method.to_string().as_str(), args.as_slice()) { + ("add_filter", [step]) => { + StepPlan::Filter(Self::step(&run.sources, &source.path, step, Role::Filter)?) + } + ("add_tap", [step]) => { + StepPlan::Tap(Self::step(&run.sources, &source.path, step, Role::Tap)?) + } + ("add_valve", [predicate, filter]) => StepPlan::Valve( + Self::step(&run.sources, &source.path, predicate, Role::Predicate)?, + Self::step(&run.sources, &source.path, filter, Role::Filter)?, + ), + ("use_hook", _) | ("observe", _) => { + return Err(Refusal::new( + place, + "A Hook, or observe(): the compiled code has no lifecycle and no State", + "Run the pipeline itself, or take the Hook out", + ) + .into()) + } + ("add_pipeline", _) => { + return Err(Refusal::new( + place, + "A nested pipeline is not compiled yet", + "Add its steps to this pipeline", + ) + .into()) + } + (method, _) => { + return Err(Refusal::new( + place, + format!("`{method}` is not a call that adds a step"), + "Keep only the add_... calls", + ) + .into()) + } + }; + steps.push(plan); + } + run.steps = steps; + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/src/source.rs b/packages/codeupipe-compile-rs/src/source.rs new file mode 100644 index 0000000..d7a0277 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/source.rs @@ -0,0 +1,6 @@ +/// One parsed file. +#[derive(Clone)] +pub struct Source { + pub path: String, + pub tree: syn::File, +} diff --git a/packages/codeupipe-compile-rs/src/step.rs b/packages/codeupipe-compile-rs/src/step.rs new file mode 100644 index 0000000..aa85408 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/step.rs @@ -0,0 +1,30 @@ +use syn::Expr; + +use crate::place::Place; + +/// The role of a step: what its method is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Role { + Filter, + Predicate, + Tap, +} + +/// One step as the pipeline makes it: its type, and each setting with the +/// literal that the pipeline gives it. +#[derive(Clone)] +pub struct Step { + pub role: Role, + pub type_name: String, + pub settings: Vec<(String, Expr)>, + pub place: Place, +} + +/// One `add_...` call of the pipeline. +#[derive(Clone)] +pub enum StepPlan { + Filter(Step), + /// The Predicate, then the Filter. + Valve(Step, Step), + Tap(Step), +} diff --git a/packages/codeupipe-compile-rs/src/type_keys_filter.rs b/packages/codeupipe-compile-rs/src/type_keys_filter.rs new file mode 100644 index 0000000..7f32bb0 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/type_keys_filter.rs @@ -0,0 +1,100 @@ +use codeupipe::{Filter, StepError}; + +use crate::compile_run::CompileRun; +use crate::key_type::KeyType; +use crate::payload_rewriter::PayloadRewriter; +use crate::refusal::Refusal; + +/// Gives each key one type, from the reads and the writes that show one, in +/// the order of its first use. A key that two uses give two types is +/// refused. A key that is only tested (`has`) or removed is text. A key that +/// a step writes with a value whose type the code does not show, and that +/// no use gives a type, is refused. +pub struct TypeKeysFilter; + +impl Filter for TypeKeysFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + let mut keys: Vec<(String, String, Option, Option)> = Vec::new(); + for (at, key_use) in run.uses.iter().enumerate() { + let known = keys.iter().position(|(key, ..)| *key == key_use.key); + let index = match known { + Some(index) => index, + None => { + keys.push(( + key_use.key.clone(), + PayloadRewriter::field_of(&key_use.key), + None, + None, + )); + keys.len() - 1 + } + }; + let entry = &mut keys[index]; + if key_use.untyped_write && entry.3.is_none() { + entry.3 = Some(at); + } + let Some(key_type) = key_use.key_type else { + continue; + }; + match entry.2 { + None => entry.2 = Some(key_type), + Some(before) if before != key_type => { + let first = run + .uses + .iter() + .find(|u| u.key == key_use.key && u.key_type == Some(before)) + .expect("the use that gave the first type"); + let what = format!( + "The key \"{}\" is {} in {} and {} in {}", + key_use.key, + before.name(), + first.step, + key_type.name(), + key_use.step + ); + return Err(Refusal::new( + key_use.place.clone(), + what, + "Give each key one type", + ) + .into()); + } + Some(_) => {} + } + } + for (key, field, key_type, untyped) in &keys { + if key_type.is_none() { + if let Some(at) = untyped { + let key_use = &run.uses[*at]; + let what = format!("The type of the key \"{key}\" cannot be told"); + return Err(Refusal::new( + key_use.place.clone(), + what, + "Read the key once with .and_then(Value::as_...), or write a literal", + ) + .into()); + } + } + if let Some(other) = keys + .iter() + .find(|(other, other_field, ..)| other != key && other_field == field) + { + let what = format!( + "The keys \"{key}\" and \"{}\" give the same field `{field}`", + other.0 + ); + return Err(Refusal::new( + run.uses[0].place.clone(), + what, + "Give the keys names that differ in letters, digits or _", + ) + .into()); + } + } + run.keys = keys + .into_iter() + .map(|(key, field, key_type, _)| (key, field, key_type.unwrap_or(KeyType::Text))) + .collect(); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/compile.rs b/packages/codeupipe-compile-rs/tests/compile.rs new file mode 100644 index 0000000..c8440fc --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/compile.rs @@ -0,0 +1,170 @@ +// The compiler writes the modules of tests/expected/ from the pipelines of +// tests/fixtures/, and refuses each shape that it cannot keep the same, with +// the place. UPDATE_EXPECTED=1 writes the expected files again; review them. +use std::fs; +use std::path::Path; + +use codeupipe_compile::compile; + +fn expect_module(fixture: &str, pipeline: &str, expected: &str) { + let written = compile(&[format!("tests/fixtures/{fixture}")], pipeline) + .unwrap_or_else(|refusal| panic!("{refusal}")); + let path = format!("tests/expected/{expected}"); + if std::env::var("UPDATE_EXPECTED").is_ok() { + fs::write(&path, &written).unwrap(); + } + // The code is compared, not its layout: cargo fmt can format the expected file in its own way. + let same_layout = |text: &str| prettyplease::unparse(&syn::parse_file(text).unwrap()); + assert_eq!( + same_layout(&written), + same_layout(&fs::read_to_string(&path).unwrap()), + "{path}" + ); +} + +#[test] +fn the_pricing_pipeline_compiles_to_the_expected_module() { + expect_module("pricing", "PricingPipeline", "pricing.rs"); +} + +#[test] +fn the_signup_pipeline_compiles_to_the_expected_module() { + expect_module("signup", "SignupPipeline", "signup.rs"); +} + +/// A project of one step file and a pipeline file, in a new directory. The step is `step` added with `add`. +fn refused(step: &str, add: &str) -> String { + let directory = + std::env::temp_dir().join(format!("cup-compile-{}-{}", std::process::id(), fastrand())); + fs::create_dir_all(&directory).unwrap(); + fs::write(directory.join("step.rs"), step).unwrap(); + fs::write( + directory.join("the_pipeline.rs"), + format!("use codeupipe::{{NamedPipeline, Pipeline}};\npub struct ThePipeline;\nimpl NamedPipeline for ThePipeline {{\n fn add_steps(&self, pipeline: &mut Pipeline) {{\n {add}\n }}\n}}\n"), + ) + .unwrap(); + let out = compile(&[directory.to_string_lossy().to_string()], "ThePipeline"); + fs::remove_dir_all(&directory).unwrap(); + match out { + Ok(module) => panic!("compiled, and should be refused:\n{module}"), + Err(refusal) => refusal.to_string(), + } +} + +/// A number for each project, so that tests that run at the same time have directories of their own. +fn fastrand() -> usize { + static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) +} + +const HEAD: &str = "use codeupipe::{Filter, Hook, Payload, Predicate, StepError, Value};\npub struct KeyFilter { key: String }\n"; + +fn assert_refused(message: &str, what: &str, place: &str) { + assert!(message.contains(what), "{message}"); + assert!(message.contains(place), "{message}"); +} + +#[test] +fn a_hook_is_refused() { + let step = "use codeupipe::{Hook, Payload};\npub struct LogHook;\nimpl Hook for LogHook {}\n"; + assert_refused( + &refused(step, "pipeline.use_hook(LogHook);"), + "A Hook", + "the_pipeline.rs:5", + ); +} + +#[test] +fn a_key_that_is_not_text_in_the_code_is_refused() { + let step = format!("{HEAD}pub struct AnyFilter;\nimpl Filter for AnyFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n let key = format!(\"k{{}}\", 1);\n payload.insert(key, 1.0);\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused(&step, "pipeline.add_filter(AnyFilter);"), + "A key that is not text in the code", + "step.rs:7", + ); +} + +#[test] +fn a_key_with_two_types_is_refused() { + let step = format!("{HEAD}pub struct AFilter;\nimpl Filter for AFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n payload.insert(\"n\", 1.0);\n Ok(())\n }}\n}}\npub struct BFilter;\nimpl Filter for BFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n payload.insert(\"n\", true);\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused( + &step, + "pipeline.add_filter(AFilter);\n pipeline.add_filter(BFilter);", + ), + "The key \"n\" is a number", + "step.rs:13", + ); +} + +#[test] +fn the_payload_given_to_other_code_is_refused() { + let step = format!("{HEAD}pub struct PrintFilter;\nimpl Filter for PrintFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n println!(\"{{:?}}\", payload.to_dict());\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused(&step, "pipeline.add_filter(PrintFilter);"), + "payload.to_dict()", + "step.rs:6", + ); +} + +#[test] +fn a_read_with_no_type_is_refused() { + let step = format!("{HEAD}pub struct MatchFilter;\nimpl Filter for MatchFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n let n = match payload.get(\"n\") {{ Some(Value::Number(n)) => *n, _ => 0.0 }};\n payload.insert(\"n\", n + 1.0);\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused(&step, "pipeline.add_filter(MatchFilter);"), + "and_then(Value::as_f64)", + "step.rs:6", + ); +} + +#[test] +fn a_setting_that_the_constructor_computes_is_refused() { + let step = format!("{HEAD}pub struct HalfFilter {{ rate: f64 }}\nimpl HalfFilter {{\n pub fn new(rate: f64) -> Self {{\n HalfFilter {{ rate: rate / 2.0 }}\n }}\n}}\nimpl Filter for HalfFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n payload.insert(\"rate\", self.rate);\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused(&step, "pipeline.add_filter(HalfFilter::new(1.0));"), + "computes its setting", + "step.rs:6", + ); +} + +#[test] +fn a_setting_that_is_not_a_literal_is_refused() { + let step = format!("{HEAD}pub struct RateFilter {{ rate: f64 }}\nimpl RateFilter {{\n pub fn new(rate: f64) -> Self {{\n RateFilter {{ rate }}\n }}\n}}\nimpl Filter for RateFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n payload.insert(\"rate\", self.rate);\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused( + &step, + "let rate = 0.5;\n pipeline.add_filter(RateFilter::new(rate));", + ), + "not a literal", + "the_pipeline.rs:6", + ); +} + +#[test] +fn a_step_with_no_source_is_refused() { + let step = HEAD.to_string(); + assert_refused( + &refused(&step, "pipeline.add_filter(ElsewhereFilter);"), + "ElsewhereFilter", + "the_pipeline.rs:5", + ); +} + +#[test] +fn a_step_that_calls_a_method_of_its_own_is_refused() { + let step = format!("{HEAD}pub struct HelpFilter;\nimpl HelpFilter {{\n fn rate(&self) -> f64 {{ 0.5 }}\n}}\nimpl Filter for HelpFilter {{\n fn call(&self, payload: &mut Payload) -> Result<(), StepError> {{\n payload.insert(\"rate\", self.rate());\n Ok(())\n }}\n}}\n"); + assert_refused( + &refused(&step, "pipeline.add_filter(HelpFilter);"), + "self.rate()", + "step.rs:9", + ); +} + +#[test] +fn the_compiler_refuses_a_pipeline_that_is_not_there() { + let message = compile(&["tests/fixtures/pricing".to_string()], "NoSuchPipeline") + .unwrap_err() + .to_string(); + assert!(message.contains("NoSuchPipeline"), "{message}"); + assert!(Path::new("tests/fixtures/pricing").exists()); +} diff --git a/packages/codeupipe-compile-rs/tests/expected/pricing.rs b/packages/codeupipe-compile-rs/tests/expected/pricing.rs new file mode 100644 index 0000000..9e1f66b --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/expected/pricing.rs @@ -0,0 +1,49 @@ +//! Written by codeupipe-compile from PricingPipeline. Do not edit it: compile again. +//! +//! Plain Rust, with no codeupipe: a struct with one field for each key that +//! the steps use, and one function for each step. It has no State and no +//! Hooks. A Tap failure does not stop the run. +/// The error of a step. +pub type StepError = Box; +/// The data of PricingPipeline: one field for each key that its steps use. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PricingData { + /// The key "price". + pub price: Option, + /// The key "qty". + pub qty: Option, + /// The key "bulk". + pub bulk: Option, +} +/// DiscountFilter +fn cup_01_discount_filter(payload: &mut PricingData) -> Result<(), StepError> { + let price = payload.price.unwrap_or(0.0) * (1.0 - 0.1); + payload.price = Some(::core::convert::Into::into((price * 100.0).round() / 100.0)); + Ok(()) +} +/// AtLeastPredicate +fn cup_02_when_at_least_predicate(payload: &PricingData) -> bool { + payload.qty.unwrap_or(0.0) >= 10.0 +} +/// MarkBulkFilter +fn cup_02_mark_bulk_filter(payload: &mut PricingData) -> Result<(), StepError> { + payload.bulk = Some(::core::convert::Into::into(true)); + Ok(()) +} +/// AuditTap +fn cup_03_audit_tap(payload: &PricingData) -> Result<(), StepError> { + if payload.price.unwrap_or(0.0) < 0.0 { + return Err("the price is below zero".into()); + } + Ok(()) +} +/// Runs the steps of PricingPipeline, in order, on a copy of `input`. +pub fn run(input: &PricingData) -> Result { + let mut payload = input.clone(); + cup_01_discount_filter(&mut payload)?; + if cup_02_when_at_least_predicate(&payload) { + cup_02_mark_bulk_filter(&mut payload)?; + } + let _ = cup_03_audit_tap(&payload); + Ok(payload) +} diff --git a/packages/codeupipe-compile-rs/tests/expected/signup.rs b/packages/codeupipe-compile-rs/tests/expected/signup.rs new file mode 100644 index 0000000..1a2eb71 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/expected/signup.rs @@ -0,0 +1,65 @@ +//! Written by codeupipe-compile from SignupPipeline. Do not edit it: compile again. +//! +//! Plain Rust, with no codeupipe: a struct with one field for each key that +//! the steps use, and one function for each step. It has no State and no +//! Hooks. A Tap failure does not stop the run. +/// The error of a step. +pub type StepError = Box; +/// The data of SignupPipeline: one field for each key that its steps use. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SignupData { + /// The key "email". + pub email: Option, + /// The key "welcome". + pub welcome: Option, + /// The key "message". + pub message: Option, + /// The key "coupon". + pub coupon: Option, + /// The key "discount". + pub discount: Option, +} +/// NormalizeEmailFilter +fn cup_01_normalize_email_filter(payload: &mut SignupData) -> Result<(), StepError> { + let email = payload.email.as_deref().unwrap_or("").trim().to_lowercase(); + if email.is_empty() { + return Err("there is no email".into()); + } + payload.email = Some(::core::convert::Into::into(email)); + Ok(()) +} +/// WelcomeFilter +fn cup_02_welcome_filter(payload: &mut SignupData) -> Result<(), StepError> { + payload.welcome = Some(::core::convert::Into::into(true)); + payload.message = Some(::core::convert::Into::into(format!( + "{}!", + String::from("Welcome") + ))); + Ok(()) +} +/// HasKeyPredicate +fn cup_03_when_has_key_predicate(payload: &SignupData) -> bool { + payload.coupon.is_some() +} +/// ApplyCouponFilter +fn cup_03_apply_coupon_filter(payload: &mut SignupData) -> Result<(), StepError> { + payload.coupon = None; + payload.discount = Some(::core::convert::Into::into(5.0)); + Ok(()) +} +/// CountTap +fn cup_04_count_tap(payload: &SignupData) -> Result<(), StepError> { + std::hint::black_box(payload.welcome); + Ok(()) +} +/// Runs the steps of SignupPipeline, in order, on a copy of `input`. +pub fn run(input: &SignupData) -> Result { + let mut payload = input.clone(); + cup_01_normalize_email_filter(&mut payload)?; + cup_02_welcome_filter(&mut payload)?; + if cup_03_when_has_key_predicate(&payload) { + cup_03_apply_coupon_filter(&mut payload)?; + } + let _ = cup_04_count_tap(&payload); + Ok(payload) +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/at_least_predicate.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/at_least_predicate.rs new file mode 100644 index 0000000..b7e96ed --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/at_least_predicate.rs @@ -0,0 +1,25 @@ +use codeupipe::{Payload, Predicate, Value}; + +pub struct AtLeastPredicate { + key: String, + minimum: f64, +} + +impl AtLeastPredicate { + pub fn new(key: &str, minimum: f64) -> Self { + AtLeastPredicate { + key: key.to_string(), + minimum, + } + } +} + +impl Predicate for AtLeastPredicate { + fn test(&self, payload: &Payload) -> bool { + payload + .get(&self.key) + .and_then(Value::as_f64) + .unwrap_or(0.0) + >= self.minimum + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/audit_tap.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/audit_tap.rs new file mode 100644 index 0000000..e073b2c --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/audit_tap.rs @@ -0,0 +1,13 @@ +use codeupipe::{Payload, StepError, Tap, Value}; + +/// Reads the price. A negative price is an error of the Tap, which does not stop the run. +pub struct AuditTap; + +impl Tap for AuditTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + if payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) < 0.0 { + return Err("the price is below zero".into()); + } + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/discount_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/discount_filter.rs new file mode 100644 index 0000000..bcf56cb --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/discount_filter.rs @@ -0,0 +1,19 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +pub struct DiscountFilter { + rate: f64, +} + +impl DiscountFilter { + pub fn new(rate: f64) -> Self { + DiscountFilter { rate } + } +} + +impl Filter for DiscountFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let price = payload.get("price").and_then(Value::as_f64).unwrap_or(0.0) * (1.0 - self.rate); + payload.insert("price", (price * 100.0).round() / 100.0); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/mark_bulk_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/mark_bulk_filter.rs new file mode 100644 index 0000000..22ce70d --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/mark_bulk_filter.rs @@ -0,0 +1,10 @@ +use codeupipe::{Filter, Payload, StepError}; + +pub struct MarkBulkFilter; + +impl Filter for MarkBulkFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("bulk", true); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/mod.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/mod.rs new file mode 100644 index 0000000..18470ca --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/mod.rs @@ -0,0 +1,5 @@ +pub mod at_least_predicate; +pub mod audit_tap; +pub mod discount_filter; +pub mod mark_bulk_filter; +pub mod pricing_pipeline; diff --git a/packages/codeupipe-compile-rs/tests/fixtures/pricing/pricing_pipeline.rs b/packages/codeupipe-compile-rs/tests/fixtures/pricing/pricing_pipeline.rs new file mode 100644 index 0000000..4dd949d --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/pricing/pricing_pipeline.rs @@ -0,0 +1,16 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use super::at_least_predicate::AtLeastPredicate; +use super::audit_tap::AuditTap; +use super::discount_filter::DiscountFilter; +use super::mark_bulk_filter::MarkBulkFilter; + +pub struct PricingPipeline; + +impl NamedPipeline for PricingPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(DiscountFilter::new(0.1)); + pipeline.add_valve(AtLeastPredicate::new("qty", 10.0), MarkBulkFilter); + pipeline.add_tap(AuditTap); + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/apply_coupon_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/apply_coupon_filter.rs new file mode 100644 index 0000000..9cb8b37 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/apply_coupon_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError}; + +/// Turns a coupon into a discount, and takes the coupon away. +pub struct ApplyCouponFilter { + amount: f64, +} + +impl ApplyCouponFilter { + pub fn new(amount: f64) -> Self { + ApplyCouponFilter { amount } + } +} + +impl Filter for ApplyCouponFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.remove("coupon"); + payload.insert("discount", self.amount); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/count_tap.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/count_tap.rs new file mode 100644 index 0000000..3d6ecd9 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/count_tap.rs @@ -0,0 +1,11 @@ +use codeupipe::{Payload, StepError, Tap, Value}; + +/// Reads the welcome flag, as a counter would. +pub struct CountTap; + +impl Tap for CountTap { + fn observe(&self, payload: &Payload) -> Result<(), StepError> { + std::hint::black_box(payload.get("welcome").and_then(Value::as_bool)); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/has_key_predicate.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/has_key_predicate.rs new file mode 100644 index 0000000..8eb51d2 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/has_key_predicate.rs @@ -0,0 +1,17 @@ +use codeupipe::{Payload, Predicate}; + +pub struct HasKeyPredicate { + key: String, +} + +impl HasKeyPredicate { + pub fn new(key: &str) -> Self { + HasKeyPredicate { key: key.into() } + } +} + +impl Predicate for HasKeyPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.has(&self.key) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/mod.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/mod.rs new file mode 100644 index 0000000..28b4844 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/mod.rs @@ -0,0 +1,6 @@ +pub mod apply_coupon_filter; +pub mod count_tap; +pub mod has_key_predicate; +pub mod normalize_email_filter; +pub mod signup_pipeline; +pub mod welcome_filter; diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/normalize_email_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/normalize_email_filter.rs new file mode 100644 index 0000000..9dfbec0 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/normalize_email_filter.rs @@ -0,0 +1,20 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// Trims the email and makes it lower case. With no email, the run fails. +pub struct NormalizeEmailFilter; + +impl Filter for NormalizeEmailFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let email = payload + .get("email") + .and_then(Value::as_str) + .unwrap_or("") + .trim() + .to_lowercase(); + if email.is_empty() { + return Err("there is no email".into()); + } + payload.insert("email", email); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/signup_pipeline.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/signup_pipeline.rs new file mode 100644 index 0000000..a323d32 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/signup_pipeline.rs @@ -0,0 +1,18 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use super::apply_coupon_filter::ApplyCouponFilter; +use super::count_tap::CountTap; +use super::has_key_predicate::HasKeyPredicate; +use super::normalize_email_filter::NormalizeEmailFilter; +use super::welcome_filter::WelcomeFilter; + +pub struct SignupPipeline; + +impl NamedPipeline for SignupPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(NormalizeEmailFilter); + pipeline.add_filter(WelcomeFilter::new("Welcome")); + pipeline.add_valve(HasKeyPredicate::new("coupon"), ApplyCouponFilter::new(5.0)); + pipeline.add_tap(CountTap); + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/signup/welcome_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/signup/welcome_filter.rs new file mode 100644 index 0000000..cd7f675 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/signup/welcome_filter.rs @@ -0,0 +1,21 @@ +use codeupipe::{Filter, Payload, StepError}; + +pub struct WelcomeFilter { + text: String, +} + +impl WelcomeFilter { + pub fn new(text: &str) -> Self { + WelcomeFilter { + text: String::from(text), + } + } +} + +impl Filter for WelcomeFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + payload.insert("welcome", true); + payload.insert("message", format!("{}!", self.text)); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/same_output.rs b/packages/codeupipe-compile-rs/tests/same_output.rs new file mode 100644 index 0000000..76d3016 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/same_output.rs @@ -0,0 +1,98 @@ +// The compiled modules in tests/expected/ give the output of the pipelines +// that they come from: the same keys and values, and the same error. +#[path = "fixtures/pricing/mod.rs"] +mod pricing; +#[path = "expected/pricing.rs"] +mod pricing_compiled; +#[path = "fixtures/signup/mod.rs"] +mod signup; +#[path = "expected/signup.rs"] +mod signup_compiled; + +use codeupipe::{Payload, Pipeline, Value}; + +fn text(payload: &Payload, key: &str) -> Option { + payload.get(key).and_then(Value::as_str).map(String::from) +} + +#[test] +fn the_compiled_pricing_gives_the_output_of_the_pipeline() { + let pipeline = Pipeline::named(pricing::pricing_pipeline::PricingPipeline); + let inputs = [ + (Some(100.0), Some(12.0)), + (Some(100.0), Some(3.0)), + (Some(19.99), Some(10.0)), + (Some(-5.0), Some(50.0)), + (None, None), + (Some(7.5), None), + ]; + for (price, qty) in inputs { + let mut payload = Payload::new(); + if let Some(price) = price { + payload.insert("price", price); + } + if let Some(qty) = qty { + payload.insert("qty", qty); + } + let data = pricing_compiled::PricingData { + price, + qty, + bulk: None, + }; + let dynamic = pipeline.run(&payload).map_err(|error| error.to_string()); + let compiled = pricing_compiled::run(&data).map_err(|error| error.to_string()); + let dynamic = dynamic.map(|out| pricing_compiled::PricingData { + price: out.get("price").and_then(Value::as_f64), + qty: out.get("qty").and_then(Value::as_f64), + bulk: out.get("bulk").and_then(Value::as_bool), + }); + assert_eq!(compiled, dynamic, "price {price:?}, qty {qty:?}"); + } +} + +#[test] +fn the_compiled_signup_gives_the_output_of_the_pipeline_and_the_same_error() { + let pipeline = Pipeline::named(signup::signup_pipeline::SignupPipeline); + let inputs = [ + (Some(" Ada@Example.COM "), Some("SAVE5")), + (Some("bob@example.com"), None), + (Some(" "), Some("X")), + (None, None), + ]; + for (email, coupon) in inputs { + let mut payload = Payload::new(); + if let Some(email) = email { + payload.insert("email", email); + } + if let Some(coupon) = coupon { + payload.insert("coupon", coupon); + } + let data = signup_compiled::SignupData { + email: email.map(String::from), + coupon: coupon.map(String::from), + ..Default::default() + }; + let dynamic = pipeline.run(&payload).map_err(|error| error.to_string()); + let compiled = signup_compiled::run(&data).map_err(|error| error.to_string()); + let dynamic = dynamic.map(|out| signup_compiled::SignupData { + email: text(&out, "email"), + welcome: out.get("welcome").and_then(Value::as_bool), + message: text(&out, "message"), + coupon: text(&out, "coupon"), + discount: out.get("discount").and_then(Value::as_f64), + }); + assert_eq!(compiled, dynamic, "email {email:?}, coupon {coupon:?}"); + } +} + +#[test] +fn the_input_is_not_changed() { + let data = pricing_compiled::PricingData { + price: Some(100.0), + qty: Some(12.0), + bulk: None, + }; + let before = data.clone(); + pricing_compiled::run(&data).unwrap(); + assert_eq!(data, before); +} From 2fca7f8634fa659cf8c08c5e0a77e2948866931e Mon Sep 17 00:00:00 2001 From: Joshua Wink Date: Wed, 28 Oct 2026 23:03:45 -0500 Subject: [PATCH 2/2] perf(rust): compiled code does not copy the caller's text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a key is text, the compiled steps work on a second struct that borrows the caller's text (SignupWork, a Cow for each text field). A step that writes text owns it, and run copies into the output only the caller's text that no step wrote. A String setting is made once, by a function of its own with a OnceLock, and not at each run. The signature of run does not change. A pipeline with no text key is compiled as before. The compiler gets a sixth Filter for this, BorrowTextFilter, after the types of the keys are known. Signup: 0.164 to 0.086 µs, 5.9x faster than Pipeline::run (it was 3.1x). A new greet fixture gives back the caller's note when its Valve does not run, and tests/allocations.rs checks that a run allocates only for the text that its steps make and the caller's text that it gives back. --keep, to compute only the keys that the caller keeps, waits in #25. Co-Authored-By: Claude Opus 5.5 --- docs/rust-plan.md | 7 ++ packages/codeupipe-compile-rs/README.md | 25 ++++-- .../src/borrow_text_filter.rs | 53 ++++++++++++ .../src/compile_pipeline.rs | 2 + .../codeupipe-compile-rs/src/compile_run.rs | 5 ++ .../src/emit_module_filter.rs | 84 +++++++++++++++++-- packages/codeupipe-compile-rs/src/lib.rs | 2 + .../src/lower_steps_filter.rs | 4 + .../src/payload_rewriter.rs | 39 ++++++++- .../src/text_write_rewriter.rs | 46 ++++++++++ .../codeupipe-compile-rs/tests/allocations.rs | 83 ++++++++++++++++++ .../codeupipe-compile-rs/tests/compile.rs | 5 ++ .../tests/expected/greet.rs | 68 +++++++++++++++ .../tests/expected/signup.rs | 52 +++++++++--- .../tests/fixtures/greet/greet_pipeline.rs | 15 ++++ .../tests/fixtures/greet/mod.rs | 4 + .../tests/fixtures/greet/note_filter.rs | 14 ++++ .../tests/fixtures/greet/trim_name_filter.rs | 17 ++++ .../tests/fixtures/greet/vip_predicate.rs | 10 +++ .../codeupipe-compile-rs/tests/same_output.rs | 70 ++++++++++++++-- 20 files changed, 571 insertions(+), 34 deletions(-) create mode 100644 packages/codeupipe-compile-rs/src/borrow_text_filter.rs create mode 100644 packages/codeupipe-compile-rs/src/text_write_rewriter.rs create mode 100644 packages/codeupipe-compile-rs/tests/allocations.rs create mode 100644 packages/codeupipe-compile-rs/tests/expected/greet.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/greet/greet_pipeline.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/greet/mod.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/greet/note_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/greet/trim_name_filter.rs create mode 100644 packages/codeupipe-compile-rs/tests/fixtures/greet/vip_predicate.rs diff --git a/docs/rust-plan.md b/docs/rust-plan.md index 478e2f6..2aac753 100644 --- a/docs/rust-plan.md +++ b/docs/rust-plan.md @@ -445,6 +445,13 @@ refusal with the place, a codeupipe pipeline itself. It is in `packages/codeupipe-compile-rs/` ([README](../packages/codeupipe-compile-rs/README.md)). Not in it yet: the conformance target `rust:static`, and nested pipelines. +The next gain was measured by hand before it was built: the compiled code +reads the caller's text where it is, and copies only the text that no step +writes, so signup went from 3.1 to 5.9 times faster than `Pipeline::run`. +Computing only the keys that the caller keeps would gain more where an +output that nobody keeps is expensive. It waits for a real pipeline that +shows the need: [#25](https://github.com/orchestrate-solutions/codeupipe/issues/25). + ### The value type in compiled code Plain Python has `dict`, and plain TypeScript has `Map`. Plain Rust has no diff --git a/packages/codeupipe-compile-rs/README.md b/packages/codeupipe-compile-rs/README.md index c1c45c7..4930fb3 100644 --- a/packages/codeupipe-compile-rs/README.md +++ b/packages/codeupipe-compile-rs/README.md @@ -66,6 +66,13 @@ gives is not changed. The body of each step is its own code, with each key of the Payload as a field and each setting as its literal, so the Rust compiler sees all of it. +**Text is not copied.** When a key is text, the steps work on a second +struct that borrows the caller's text (`SignupWork`, with a `Cow` for each +text field). A step that writes text owns it. At the end, `run` copies into +the output only the caller's text that no step wrote. A `String` setting is +made once, by a function of its own, and not at each run. See +[tests/expected/signup.rs](tests/expected/signup.rs). + ## What compiles | Shape | Plain form | @@ -74,7 +81,7 @@ compiler sees all of it. | Filter (`call`) | A function on `&mut PData` | | Valve | `if predicate(&payload) { filter(&mut payload)?; }` | | Tap (`observe`) | `let _ = tap(&payload);`: a failure does not stop the run | -| A step made as `T`, `T { field: literal }` or `T::new(literal, ...)` | Each field of the step is a constant, from the literal. `new` may keep a parameter as it is, or with `.to_string()`, `.into()`, `.to_owned()` or `String::from(..)` | +| A step made as `T`, `T { field: literal }` or `T::new(literal, ...)` | Each field of the step is a constant, from the literal; a `String` field is made once. `new` may keep a parameter as it is, or with `.to_string()`, `.into()`, `.to_owned()` or `String::from(..)` | | `payload.get(K).and_then(Value::as_f64)` (or `as_bool`, `as_str`, or a closure `\|v\| v.as_f64()`) | `payload.k` (`payload.k.as_deref()` for text) | | `payload.has(K)` | `payload.k.is_some()` | | `payload.insert(K, v)` | `payload.k = Some(v.into())` | @@ -133,19 +140,25 @@ two alternated): | Pipeline | `Pipeline::run` | Compiled | | |---|---|---|---| -| pricing: 3 steps of arithmetic | 0.237 µs | 0.0016 µs | 150x | -| signup: 4 steps of text (trim, lower case, format) | 0.504 µs | 0.164 µs | 3.1x | +| pricing: 3 steps of arithmetic | 0.237 µs | 0.0017 µs | 140x | +| signup: 4 steps of text (trim, lower case, format) | 0.50 µs | 0.086 µs | 5.9x | The more a step does, the less the pipeline costs next to it: with steps of real work the gain is small. The large gain is for many runs of light -steps. +steps. Before the compiled code borrowed the caller's text, signup ran in +0.164 µs (3.1x): it copied the email and the coupon, and made the setting +"Welcome", at each run. ## How it is checked - [tests/same_output.rs](tests/same_output.rs): the modules of [tests/expected/](tests/expected/) are compiled into the test and give the output and the errors of the pipelines that they come from, on many - inputs, and do not change their input. + inputs, and do not change their input. The greet fixture gives back the + caller's note when its Valve does not run. +- [tests/allocations.rs](tests/allocations.rs): a compiled run allocates + only for the text that its steps make, and for the caller's text that it + gives back as it is. - [tests/compile.rs](tests/compile.rs): the compiler writes those modules (the code is compared, not its layout; `UPDATE_EXPECTED=1` writes them again), and refuses each shape above with its place. @@ -156,6 +169,8 @@ steps. `conformance/cases/` as `typescript:static` is. - Nested pipelines. - A `from_map`/`to_map` for a caller that has a Payload. +- `--keep`, to compute only the keys that the caller keeps + ([#25](https://github.com/orchestrate-solutions/codeupipe/issues/25)). ## Development diff --git a/packages/codeupipe-compile-rs/src/borrow_text_filter.rs b/packages/codeupipe-compile-rs/src/borrow_text_filter.rs new file mode 100644 index 0000000..002c462 --- /dev/null +++ b/packages/codeupipe-compile-rs/src/borrow_text_filter.rs @@ -0,0 +1,53 @@ +use codeupipe::{Filter, StepError}; +use quote::format_ident; +use syn::visit_mut::VisitMut; +use syn::{FnArg, Pat}; + +use crate::compile_run::CompileRun; +use crate::key_type::KeyType; +use crate::step::Role; +use crate::text_write_rewriter::TextWriteRewriter; + +/// When a key is text, the steps work on a struct that borrows the caller's +/// text (`SignupWork`), so that `run` does not copy text that a step writes +/// over. A write of text owns its text. With no text key, the steps work on +/// the struct of the data, which is cheap to copy. +pub struct BorrowTextFilter; + +impl Filter for BorrowTextFilter { + fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { + let text: Vec = run + .keys + .iter() + .filter(|(_, _, key_type)| *key_type == KeyType::Text) + .map(|(_, field, _)| field.clone()) + .collect(); + if text.is_empty() { + return Ok(()); + } + let work = format!("{}Work", run.data.strip_suffix("Data").unwrap_or(&run.data)); + let work_type = format_ident!("{}", work); + for lowered in &mut run.functions { + let Some(FnArg::Typed(param)) = lowered.item.sig.inputs.first_mut() else { + continue; + }; + let Pat::Ident(payload) = &*param.pat else { + continue; + }; + let payload = payload.ident.to_string(); + *param.ty = match lowered.role { + Role::Filter => syn::parse_quote!(&mut #work_type<'_>), + Role::Predicate | Role::Tap => syn::parse_quote!(&#work_type<'_>), + }; + if lowered.role == Role::Filter { + TextWriteRewriter { + payload, + fields: &text, + } + .visit_block_mut(&mut lowered.item.block); + } + } + run.work = Some(work); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/src/compile_pipeline.rs b/packages/codeupipe-compile-rs/src/compile_pipeline.rs index 924a87a..722eb84 100644 --- a/packages/codeupipe-compile-rs/src/compile_pipeline.rs +++ b/packages/codeupipe-compile-rs/src/compile_pipeline.rs @@ -1,5 +1,6 @@ use codeupipe::{NamedPipeline, Pipeline}; +use crate::borrow_text_filter::BorrowTextFilter; use crate::compile_run::CompileRun; use crate::emit_module_filter::EmitModuleFilter; use crate::lower_steps_filter::LowerStepsFilter; @@ -16,6 +17,7 @@ impl NamedPipeline for CompilePipeline { pipeline.add_filter(ResolvePipelineFilter); // the pipeline to a list of steps, with their settings pipeline.add_filter(LowerStepsFilter); // each step to a plain function on the struct pipeline.add_filter(TypeKeysFilter); // each key to a field with one type + pipeline.add_filter(BorrowTextFilter); // with text keys, the steps borrow the caller's text pipeline.add_filter(EmitModuleFilter); // the struct and the functions to one module } } diff --git a/packages/codeupipe-compile-rs/src/compile_run.rs b/packages/codeupipe-compile-rs/src/compile_run.rs index 691035a..832ddd9 100644 --- a/packages/codeupipe-compile-rs/src/compile_run.rs +++ b/packages/codeupipe-compile-rs/src/compile_run.rs @@ -26,8 +26,13 @@ pub struct CompileRun { pub uses: Vec, /// The `use` items of the step files that the functions need (LowerStepsFilter). pub imports: Vec, + /// A function for each text setting, which makes it once (LowerStepsFilter). + pub constants: Vec, /// Each key with its field and its type, in the order of first use (TypeKeysFilter). pub keys: Vec<(String, String, KeyType)>, + /// The struct that the steps work on when a key is text: `SignupWork`, which borrows the caller's + /// text. With no text, the steps work on the struct of the data (BorrowTextFilter). + pub work: Option, /// The compiled module (EmitModuleFilter). pub module: String, } diff --git a/packages/codeupipe-compile-rs/src/emit_module_filter.rs b/packages/codeupipe-compile-rs/src/emit_module_filter.rs index 38ff583..59bbcd2 100644 --- a/packages/codeupipe-compile-rs/src/emit_module_filter.rs +++ b/packages/codeupipe-compile-rs/src/emit_module_filter.rs @@ -2,12 +2,70 @@ use codeupipe::{Filter, StepError}; use quote::{format_ident, quote}; use crate::compile_run::CompileRun; +use crate::key_type::KeyType; use crate::step::{Role, StepPlan}; -/// The compiled module: the struct of the keys, the function of each step, -/// and `run`, formatted as Rust. +/// The compiled module: the struct of the keys, the struct that the steps +/// work on (when a key is text), the function of each text setting and of +/// each step, and `run`, formatted as Rust. pub struct EmitModuleFilter; +impl EmitModuleFilter { + /// The struct that borrows the caller's text, and `run` on it: the text + /// that no step wrote is copied into the output at the end. + fn work_and_run( + run: &CompileRun, + work: &str, + calls: &[proc_macro2::TokenStream], + ) -> proc_macro2::TokenStream { + let data = format_ident!("{}", run.data); + let work = format_ident!("{}", work); + let pipeline = &run.pipeline; + let work_doc = [ + format!(" The data while the steps of {pipeline} run. Text that no step has"), + " written yet is the caller's, borrowed and not copied.".to_string(), + ]; + let run_doc = [ + format!(" Runs the steps of {pipeline}, in order. The caller's text is read where"), + " it is: only the text that no step wrote is copied into the output.".to_string(), + ]; + let mut fields = Vec::new(); + let mut from_input = Vec::new(); + let mut to_output = Vec::new(); + for (_, field, key_type) in &run.keys { + let field = format_ident!("{}", field); + if *key_type == KeyType::Text { + fields.push(quote!(#field: Option<::std::borrow::Cow<'a, str>>,)); + from_input.push( + quote!(#field: input.#field.as_deref().map(::std::borrow::Cow::Borrowed),), + ); + to_output.push(quote!(#field: payload.#field.map(::std::borrow::Cow::into_owned),)); + } else { + let ty = key_type.rust(); + fields.push(quote!(#field: Option<#ty>,)); + from_input.push(quote!(#field: input.#field,)); + to_output.push(quote!(#field: payload.#field,)); + } + } + let constants = &run.constants; + let functions = run.functions.iter().map(|lowered| &lowered.item); + quote! { + #(#[doc = #work_doc])* + struct #work<'a> { + #(#fields)* + } + #(#constants)* + #(#functions)* + #(#[doc = #run_doc])* + pub fn run(input: &#data) -> Result<#data, StepError> { + let mut payload = #work { #(#from_input)* }; + #(#calls)* + Ok(#data { #(#to_output)* }) + } + } + } +} + impl Filter for EmitModuleFilter { fn call(&self, run: &mut CompileRun) -> Result<(), StepError> { let data = format_ident!("{}", run.data); @@ -56,6 +114,20 @@ impl Filter for EmitModuleFilter { .iter() .all(|f| matches!(f.role, Role::Filter | Role::Predicate | Role::Tap))); let imports = &run.imports; + let constants = &run.constants; + let rest = match &run.work { + Some(work) => Self::work_and_run(run, work, &calls), + None => quote! { + #(#constants)* + #(#functions)* + #[doc = #run_doc] + pub fn run(input: &#data) -> Result<#data, StepError> { + let mut payload = input.clone(); + #(#calls)* + Ok(payload) + } + }, + }; let file = quote! { #(#![doc = #head])* #(#imports)* @@ -66,13 +138,7 @@ impl Filter for EmitModuleFilter { pub struct #data { #(#fields)* } - #(#functions)* - #[doc = #run_doc] - pub fn run(input: &#data) -> Result<#data, StepError> { - let mut payload = input.clone(); - #(#calls)* - Ok(payload) - } + #rest }; let tree: syn::File = syn::parse2(file)?; run.module = prettyplease::unparse(&tree); diff --git a/packages/codeupipe-compile-rs/src/lib.rs b/packages/codeupipe-compile-rs/src/lib.rs index ac9746f..07f16e6 100644 --- a/packages/codeupipe-compile-rs/src/lib.rs +++ b/packages/codeupipe-compile-rs/src/lib.rs @@ -3,6 +3,7 @@ //! no codeupipe. Code that compiles gives the output of the pipeline. A //! pipeline for which the compiler cannot keep that rule is refused, with //! the place. +mod borrow_text_filter; mod compile_pipeline; mod compile_run; mod emit_module_filter; @@ -17,6 +18,7 @@ mod refusal; mod resolve_pipeline_filter; mod source; mod step; +mod text_write_rewriter; mod type_keys_filter; pub use compile_pipeline::CompilePipeline; diff --git a/packages/codeupipe-compile-rs/src/lower_steps_filter.rs b/packages/codeupipe-compile-rs/src/lower_steps_filter.rs index 34f77e7..07e2d14 100644 --- a/packages/codeupipe-compile-rs/src/lower_steps_filter.rs +++ b/packages/codeupipe-compile-rs/src/lower_steps_filter.rs @@ -106,10 +106,13 @@ impl LowerStepsFilter { let mut rewriter = PayloadRewriter { path: &source.path, step: &step.type_name, + function: &name, payload: payload.clone(), settings: &step.settings, writes: step.role == Role::Filter, uses: Vec::new(), + constants: Vec::new(), + in_key: false, refusal: None, }; let mut block = method.block.clone(); @@ -119,6 +122,7 @@ impl LowerStepsFilter { return Err(refusal); } run.uses.extend(rewriter.uses); + run.constants.extend(rewriter.constants); let data = format_ident!("{}", run.data); let payload = format_ident!("{}", payload); let function = format_ident!("{}", name); diff --git a/packages/codeupipe-compile-rs/src/payload_rewriter.rs b/packages/codeupipe-compile-rs/src/payload_rewriter.rs index ee7ba2f..7553bbc 100644 --- a/packages/codeupipe-compile-rs/src/payload_rewriter.rs +++ b/packages/codeupipe-compile-rs/src/payload_rewriter.rs @@ -15,15 +15,23 @@ const USE_ONLY: &str = /// Rewrites the body of a step for the compiled struct: a key of the Payload /// becomes a field, and `self.setting` becomes the literal that the pipeline -/// gave it. Each other use of the Payload, of `self` or of `Value` is refused. +/// gave it. A `String` setting is made once, by a function of its own, and +/// not at each run. Each other use of the Payload, of `self` or of `Value` +/// is refused. pub struct PayloadRewriter<'a> { pub path: &'a str, pub step: &'a str, + /// The name of the compiled function of the step. + pub function: &'a str, /// The name of the Payload parameter of the step. pub payload: String, pub settings: &'a [(String, Expr)], pub writes: bool, pub uses: Vec, + /// The function of each `String` setting that the step uses. + pub constants: Vec, + /// True while the rewriter reads a key: a setting is then its literal. + pub in_key: bool, pub refusal: Option, } @@ -130,10 +138,37 @@ impl PayloadRewriter<'_> { }); } + /// The value of a setting in the function: its literal, or for a `String` setting + /// (`String::from(literal)`), the `String` that a function of its own makes once. + fn setting(&mut self, name: &str, value: &Expr) -> Expr { + let is_text = matches!(value, Expr::Call(call) if matches!(&*call.func, Expr::Path(p) if p.path.segments.len() == 2 && p.path.segments[0].ident == "String")); + if self.in_key || !is_text { + return value.clone(); + } + let function = format_ident!("{}_{}", self.function, name); + if !self.constants.iter().any(|made| made.sig.ident == function) { + let doc = format!( + " The setting `{name}` of {}, made once and not at each run.", + self.step + ); + self.constants.push(syn::parse_quote! { + #[doc = #doc] + fn #function() -> &'static String { + static VALUE: ::std::sync::OnceLock = ::std::sync::OnceLock::new(); + VALUE.get_or_init(|| #value) + } + }); + } + // A place of type String, as `self.text` is: each use of it is the same. + syn::parse_quote!((*#function())) + } + /// The text of a key: a literal, perhaps behind `&`, `.to_string()`, `String::from(..)`. Settings are put in first. fn key_of(&mut self, arg: &Expr) -> Option { let mut arg = arg.clone(); + let in_key = std::mem::replace(&mut self.in_key, true); self.visit_expr_mut(&mut arg); + self.in_key = in_key; let mut at = &arg; loop { match at { @@ -271,7 +306,7 @@ impl VisitMut for PayloadRewriter<'_> { return; }; match self.settings.iter().find(|(setting, _)| name == setting) { - Some((_, value)) => *expr = value.clone(), + Some((setting, value)) => *expr = self.setting(setting, value), None => { let span = field.span(); self.refuse( diff --git a/packages/codeupipe-compile-rs/src/text_write_rewriter.rs b/packages/codeupipe-compile-rs/src/text_write_rewriter.rs new file mode 100644 index 0000000..61b8b1a --- /dev/null +++ b/packages/codeupipe-compile-rs/src/text_write_rewriter.rs @@ -0,0 +1,46 @@ +use syn::visit_mut::{self, VisitMut}; +use syn::{Expr, Member}; + +/// Makes each write of a text key own its text: `payload.k = Some(v)` +/// becomes `payload.k = Some(Cow::Owned(v))`. The field of a text key +/// borrows the caller's text until a step writes it. +pub struct TextWriteRewriter<'a> { + /// The name of the parameter of the step. + pub payload: String, + /// The fields of the text keys. + pub fields: &'a [String], +} + +// cup: allow-plain the rewrite of a step function: a walk of its syntax tree, called by BorrowTextFilter +impl TextWriteRewriter<'_> { + /// True for `payload.k`, with `k` a text field. + fn is_text_field(&self, expr: &Expr) -> bool { + let Expr::Field(field) = expr else { + return false; + }; + let Member::Named(name) = &field.member else { + return false; + }; + matches!(&*field.base, Expr::Path(path) if path.path.is_ident(&self.payload)) + && self.fields.iter().any(|text| name == text) + } +} + +// cup: allow-plain the rewrite of a step function: a walk of its syntax tree, called by BorrowTextFilter +impl VisitMut for TextWriteRewriter<'_> { + fn visit_expr_mut(&mut self, expr: &mut Expr) { + if let Expr::Assign(assign) = expr { + if self.is_text_field(&assign.left) { + // PayloadRewriter writes a key as `Some(..)`, and removes it as `None`. + if let Expr::Call(some) = &mut *assign.right { + if let Some(value) = some.args.first_mut() { + let written = value.clone(); + *value = syn::parse_quote!(::std::borrow::Cow::Owned(#written)); + } + } + return; + } + } + visit_mut::visit_expr_mut(self, expr); + } +} diff --git a/packages/codeupipe-compile-rs/tests/allocations.rs b/packages/codeupipe-compile-rs/tests/allocations.rs new file mode 100644 index 0000000..f54a3fe --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/allocations.rs @@ -0,0 +1,83 @@ +// A compiled run allocates only for the text that its steps make and for the +// caller's text that it gives back as it is. It does not copy text that a +// step writes over, and it makes a text setting once, not at each run. +#[path = "expected/greet.rs"] +mod greet_compiled; +#[path = "expected/signup.rs"] +mod signup_compiled; + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use std::hint::black_box; + +thread_local!(static ALLOCATIONS: Cell = const { Cell::new(0) }); + +/// The allocator of this test: the system's, with a count for each thread. +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + ALLOCATIONS.with(|count| count.set(count.get() + 1)); + System.alloc(layout) + } + + unsafe fn dealloc(&self, at: *mut u8, layout: Layout) { + System.dealloc(at, layout) + } + + unsafe fn realloc(&self, at: *mut u8, layout: Layout, size: usize) -> *mut u8 { + ALLOCATIONS.with(|count| count.set(count.get() + 1)); + System.realloc(at, layout, size) + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// How many times `f` allocates. +fn allocations(f: impl FnOnce()) -> u64 { + let before = ALLOCATIONS.with(Cell::get); + f(); + ALLOCATIONS.with(Cell::get) - before +} + +#[test] +fn the_compiled_signup_allocates_only_the_text_that_its_steps_make() { + let data = signup_compiled::SignupData { + email: Some(" Ada@Example.COM ".into()), + coupon: Some("SAVE5".into()), + ..Default::default() + }; + // The first run makes the setting. + black_box(signup_compiled::run(&data).unwrap()); + let run = allocations(|| { + black_box(signup_compiled::run(black_box(&data)).unwrap()); + }); + // The email made lower case, and the message. Not a copy of the email or + // the coupon, and not the setting "Welcome". + let welcome = String::from("Welcome"); + let made = allocations(|| { + black_box(black_box(" Ada@Example.COM ").trim().to_lowercase()); + black_box(format!("{}!", black_box(&welcome))); + }); + assert_eq!(run, made); +} + +#[test] +fn the_compiled_greet_copies_only_the_callers_text_that_it_gives_back() { + let data = greet_compiled::GreetData { + name: Some(" Bob ".into()), + note: Some("table 4".into()), + ..Default::default() + }; + let run = allocations(|| { + black_box(greet_compiled::run(black_box(&data)).unwrap()); + }); + // The trimmed name, and the caller's note, which the output gives back. + // Not a copy of the name that the step writes over. + let made = allocations(|| { + black_box(black_box(" Bob ").trim().to_string()); + black_box(black_box("table 4").to_string()); + }); + assert_eq!(run, made); +} diff --git a/packages/codeupipe-compile-rs/tests/compile.rs b/packages/codeupipe-compile-rs/tests/compile.rs index c8440fc..e9e3c6f 100644 --- a/packages/codeupipe-compile-rs/tests/compile.rs +++ b/packages/codeupipe-compile-rs/tests/compile.rs @@ -32,6 +32,11 @@ fn the_signup_pipeline_compiles_to_the_expected_module() { expect_module("signup", "SignupPipeline", "signup.rs"); } +#[test] +fn the_greet_pipeline_compiles_to_the_expected_module() { + expect_module("greet", "GreetPipeline", "greet.rs"); +} + /// A project of one step file and a pipeline file, in a new directory. The step is `step` added with `add`. fn refused(step: &str, add: &str) -> String { let directory = diff --git a/packages/codeupipe-compile-rs/tests/expected/greet.rs b/packages/codeupipe-compile-rs/tests/expected/greet.rs new file mode 100644 index 0000000..14ffc86 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/expected/greet.rs @@ -0,0 +1,68 @@ +//! Written by codeupipe-compile from GreetPipeline. Do not edit it: compile again. +//! +//! Plain Rust, with no codeupipe: a struct with one field for each key that +//! the steps use, and one function for each step. It has no State and no +//! Hooks. A Tap failure does not stop the run. +/// The error of a step. +pub type StepError = Box; +/// The data of GreetPipeline: one field for each key that its steps use. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct GreetData { + /// The key "name". + pub name: Option, + /// The key "vip". + pub vip: Option, + /// The key "note". + pub note: Option, + /// The key "badge". + pub badge: Option, +} +/// The data while the steps of GreetPipeline run. Text that no step has +/// written yet is the caller's, borrowed and not copied. +struct GreetWork<'a> { + name: Option<::std::borrow::Cow<'a, str>>, + vip: Option<::std::borrow::Cow<'a, str>>, + note: Option<::std::borrow::Cow<'a, str>>, + badge: Option<::std::borrow::Cow<'a, str>>, +} +/// TrimNameFilter +fn cup_01_trim_name_filter(payload: &mut GreetWork<'_>) -> Result<(), StepError> { + let name = payload.name.as_deref().unwrap_or("").trim().to_string(); + payload.name = Some(::std::borrow::Cow::Owned(::core::convert::Into::into(name))); + Ok(()) +} +/// VipPredicate +fn cup_02_when_vip_predicate(payload: &GreetWork<'_>) -> bool { + payload.vip.is_some() +} +/// NoteFilter +fn cup_02_note_filter(payload: &mut GreetWork<'_>) -> Result<(), StepError> { + let name = payload.name.as_deref().unwrap_or(""); + payload.note = Some(::std::borrow::Cow::Owned(::core::convert::Into::into( + format!("VIP guest {name}"), + ))); + payload.badge = Some(::std::borrow::Cow::Owned(::core::convert::Into::into( + "gold", + ))); + Ok(()) +} +/// Runs the steps of GreetPipeline, in order. The caller's text is read where +/// it is: only the text that no step wrote is copied into the output. +pub fn run(input: &GreetData) -> Result { + let mut payload = GreetWork { + name: input.name.as_deref().map(::std::borrow::Cow::Borrowed), + vip: input.vip.as_deref().map(::std::borrow::Cow::Borrowed), + note: input.note.as_deref().map(::std::borrow::Cow::Borrowed), + badge: input.badge.as_deref().map(::std::borrow::Cow::Borrowed), + }; + cup_01_trim_name_filter(&mut payload)?; + if cup_02_when_vip_predicate(&payload) { + cup_02_note_filter(&mut payload)?; + } + Ok(GreetData { + name: payload.name.map(::std::borrow::Cow::into_owned), + vip: payload.vip.map(::std::borrow::Cow::into_owned), + note: payload.note.map(::std::borrow::Cow::into_owned), + badge: payload.badge.map(::std::borrow::Cow::into_owned), + }) +} diff --git a/packages/codeupipe-compile-rs/tests/expected/signup.rs b/packages/codeupipe-compile-rs/tests/expected/signup.rs index 1a2eb71..fe2e75f 100644 --- a/packages/codeupipe-compile-rs/tests/expected/signup.rs +++ b/packages/codeupipe-compile-rs/tests/expected/signup.rs @@ -19,47 +19,75 @@ pub struct SignupData { /// The key "discount". pub discount: Option, } +/// The data while the steps of SignupPipeline run. Text that no step has +/// written yet is the caller's, borrowed and not copied. +struct SignupWork<'a> { + email: Option<::std::borrow::Cow<'a, str>>, + welcome: Option, + message: Option<::std::borrow::Cow<'a, str>>, + coupon: Option<::std::borrow::Cow<'a, str>>, + discount: Option, +} +/// The setting `text` of WelcomeFilter, made once and not at each run. +fn cup_02_welcome_filter_text() -> &'static String { + static VALUE: ::std::sync::OnceLock = ::std::sync::OnceLock::new(); + VALUE.get_or_init(|| String::from("Welcome")) +} /// NormalizeEmailFilter -fn cup_01_normalize_email_filter(payload: &mut SignupData) -> Result<(), StepError> { +fn cup_01_normalize_email_filter(payload: &mut SignupWork<'_>) -> Result<(), StepError> { let email = payload.email.as_deref().unwrap_or("").trim().to_lowercase(); if email.is_empty() { return Err("there is no email".into()); } - payload.email = Some(::core::convert::Into::into(email)); + payload.email = Some(::std::borrow::Cow::Owned(::core::convert::Into::into( + email, + ))); Ok(()) } /// WelcomeFilter -fn cup_02_welcome_filter(payload: &mut SignupData) -> Result<(), StepError> { +fn cup_02_welcome_filter(payload: &mut SignupWork<'_>) -> Result<(), StepError> { payload.welcome = Some(::core::convert::Into::into(true)); - payload.message = Some(::core::convert::Into::into(format!( - "{}!", - String::from("Welcome") + payload.message = Some(::std::borrow::Cow::Owned(::core::convert::Into::into( + format!("{}!", (*cup_02_welcome_filter_text())), ))); Ok(()) } /// HasKeyPredicate -fn cup_03_when_has_key_predicate(payload: &SignupData) -> bool { +fn cup_03_when_has_key_predicate(payload: &SignupWork<'_>) -> bool { payload.coupon.is_some() } /// ApplyCouponFilter -fn cup_03_apply_coupon_filter(payload: &mut SignupData) -> Result<(), StepError> { +fn cup_03_apply_coupon_filter(payload: &mut SignupWork<'_>) -> Result<(), StepError> { payload.coupon = None; payload.discount = Some(::core::convert::Into::into(5.0)); Ok(()) } /// CountTap -fn cup_04_count_tap(payload: &SignupData) -> Result<(), StepError> { +fn cup_04_count_tap(payload: &SignupWork<'_>) -> Result<(), StepError> { std::hint::black_box(payload.welcome); Ok(()) } -/// Runs the steps of SignupPipeline, in order, on a copy of `input`. +/// Runs the steps of SignupPipeline, in order. The caller's text is read where +/// it is: only the text that no step wrote is copied into the output. pub fn run(input: &SignupData) -> Result { - let mut payload = input.clone(); + let mut payload = SignupWork { + email: input.email.as_deref().map(::std::borrow::Cow::Borrowed), + welcome: input.welcome, + message: input.message.as_deref().map(::std::borrow::Cow::Borrowed), + coupon: input.coupon.as_deref().map(::std::borrow::Cow::Borrowed), + discount: input.discount, + }; cup_01_normalize_email_filter(&mut payload)?; cup_02_welcome_filter(&mut payload)?; if cup_03_when_has_key_predicate(&payload) { cup_03_apply_coupon_filter(&mut payload)?; } let _ = cup_04_count_tap(&payload); - Ok(payload) + Ok(SignupData { + email: payload.email.map(::std::borrow::Cow::into_owned), + welcome: payload.welcome, + message: payload.message.map(::std::borrow::Cow::into_owned), + coupon: payload.coupon.map(::std::borrow::Cow::into_owned), + discount: payload.discount, + }) } diff --git a/packages/codeupipe-compile-rs/tests/fixtures/greet/greet_pipeline.rs b/packages/codeupipe-compile-rs/tests/fixtures/greet/greet_pipeline.rs new file mode 100644 index 0000000..44307c3 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/greet/greet_pipeline.rs @@ -0,0 +1,15 @@ +use codeupipe::{NamedPipeline, Pipeline}; + +use super::note_filter::NoteFilter; +use super::trim_name_filter::TrimNameFilter; +use super::vip_predicate::VipPredicate; + +/// The caller's note passes through when the guest is not a VIP. +pub struct GreetPipeline; + +impl NamedPipeline for GreetPipeline { + fn add_steps(&self, pipeline: &mut Pipeline) { + pipeline.add_filter(TrimNameFilter); + pipeline.add_valve(VipPredicate, NoteFilter); + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/greet/mod.rs b/packages/codeupipe-compile-rs/tests/fixtures/greet/mod.rs new file mode 100644 index 0000000..6a7fbf0 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/greet/mod.rs @@ -0,0 +1,4 @@ +pub mod greet_pipeline; +pub mod note_filter; +pub mod trim_name_filter; +pub mod vip_predicate; diff --git a/packages/codeupipe-compile-rs/tests/fixtures/greet/note_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/greet/note_filter.rs new file mode 100644 index 0000000..82f6efd --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/greet/note_filter.rs @@ -0,0 +1,14 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// Writes a note and a badge for a VIP. A note that the caller gave is +/// written over. +pub struct NoteFilter; + +impl Filter for NoteFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let name = payload.get("name").and_then(Value::as_str).unwrap_or(""); + payload.insert("note", format!("VIP guest {name}")); + payload.insert("badge", "gold"); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/greet/trim_name_filter.rs b/packages/codeupipe-compile-rs/tests/fixtures/greet/trim_name_filter.rs new file mode 100644 index 0000000..30bae80 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/greet/trim_name_filter.rs @@ -0,0 +1,17 @@ +use codeupipe::{Filter, Payload, StepError, Value}; + +/// Trims the name. +pub struct TrimNameFilter; + +impl Filter for TrimNameFilter { + fn call(&self, payload: &mut Payload) -> Result<(), StepError> { + let name = payload + .get("name") + .and_then(Value::as_str) + .unwrap_or("") + .trim() + .to_string(); + payload.insert("name", name); + Ok(()) + } +} diff --git a/packages/codeupipe-compile-rs/tests/fixtures/greet/vip_predicate.rs b/packages/codeupipe-compile-rs/tests/fixtures/greet/vip_predicate.rs new file mode 100644 index 0000000..fa0d264 --- /dev/null +++ b/packages/codeupipe-compile-rs/tests/fixtures/greet/vip_predicate.rs @@ -0,0 +1,10 @@ +use codeupipe::{Payload, Predicate}; + +/// True when the guest is a VIP. +pub struct VipPredicate; + +impl Predicate for VipPredicate { + fn test(&self, payload: &Payload) -> bool { + payload.has("vip") + } +} diff --git a/packages/codeupipe-compile-rs/tests/same_output.rs b/packages/codeupipe-compile-rs/tests/same_output.rs index 76d3016..a024a2f 100644 --- a/packages/codeupipe-compile-rs/tests/same_output.rs +++ b/packages/codeupipe-compile-rs/tests/same_output.rs @@ -1,5 +1,9 @@ // The compiled modules in tests/expected/ give the output of the pipelines // that they come from: the same keys and values, and the same error. +#[path = "fixtures/greet/mod.rs"] +mod greet; +#[path = "expected/greet.rs"] +mod greet_compiled; #[path = "fixtures/pricing/mod.rs"] mod pricing; #[path = "expected/pricing.rs"] @@ -53,13 +57,16 @@ fn the_compiled_pricing_gives_the_output_of_the_pipeline() { #[test] fn the_compiled_signup_gives_the_output_of_the_pipeline_and_the_same_error() { let pipeline = Pipeline::named(signup::signup_pipeline::SignupPipeline); + // The caller can give each key, also the ones that a step writes. let inputs = [ - (Some(" Ada@Example.COM "), Some("SAVE5")), - (Some("bob@example.com"), None), - (Some(" "), Some("X")), - (None, None), + (Some(" Ada@Example.COM "), Some("SAVE5"), None, None), + (Some("bob@example.com"), None, None, None), + (Some("bob@example.com"), None, Some("hi"), Some(2.0)), + (Some("cy@example.com"), Some("SAVE5"), Some("hi"), Some(2.0)), + (Some(" "), Some("X"), None, None), + (None, None, None, None), ]; - for (email, coupon) in inputs { + for (email, coupon, message, discount) in inputs { let mut payload = Payload::new(); if let Some(email) = email { payload.insert("email", email); @@ -67,9 +74,17 @@ fn the_compiled_signup_gives_the_output_of_the_pipeline_and_the_same_error() { if let Some(coupon) = coupon { payload.insert("coupon", coupon); } + if let Some(message) = message { + payload.insert("message", message); + } + if let Some(discount) = discount { + payload.insert("discount", discount); + } let data = signup_compiled::SignupData { email: email.map(String::from), coupon: coupon.map(String::from), + message: message.map(String::from), + discount, ..Default::default() }; let dynamic = pipeline.run(&payload).map_err(|error| error.to_string()); @@ -81,7 +96,50 @@ fn the_compiled_signup_gives_the_output_of_the_pipeline_and_the_same_error() { coupon: text(&out, "coupon"), discount: out.get("discount").and_then(Value::as_f64), }); - assert_eq!(compiled, dynamic, "email {email:?}, coupon {coupon:?}"); + assert_eq!( + compiled, dynamic, + "email {email:?}, coupon {coupon:?}, message {message:?}, discount {discount:?}" + ); + } +} + +#[test] +fn the_compiled_greet_gives_back_the_callers_text_that_no_step_writes() { + let pipeline = Pipeline::named(greet::greet_pipeline::GreetPipeline); + let inputs = [ + (Some(" Ada "), Some("yes"), None), + // The Valve writes over the caller's note. + (Some(" Ada "), Some("yes"), Some("an old note")), + // The Valve does not run: the caller's note is given back. + (Some("Bob"), None, Some("table 4")), + (Some("Bob"), None, None), + (None, None, None), + ]; + for (name, vip, note) in inputs { + let mut payload = Payload::new(); + for (key, value) in [("name", name), ("vip", vip), ("note", note)] { + if let Some(value) = value { + payload.insert(key, value); + } + } + let data = greet_compiled::GreetData { + name: name.map(String::from), + vip: vip.map(String::from), + note: note.map(String::from), + badge: None, + }; + let dynamic = pipeline.run(&payload).map_err(|error| error.to_string()); + let compiled = greet_compiled::run(&data).map_err(|error| error.to_string()); + let dynamic = dynamic.map(|out| greet_compiled::GreetData { + name: text(&out, "name"), + vip: text(&out, "vip"), + note: text(&out, "note"), + badge: text(&out, "badge"), + }); + assert_eq!( + compiled, dynamic, + "name {name:?}, vip {vip:?}, note {note:?}" + ); } }