diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f39455c..b58a218 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,47 @@ jobs: rustup toolchain install 1.75 --profile minimal cargo +1.75 test --locked + go: + name: Go package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + # The package and the runner have no dependencies, and no go.sum to + # cache. Only staticcheck, a tool, is downloaded. + - uses: actions/setup-go@v7 + with: + go-version: stable + cache: false + - name: Format + run: test -z "$(gofmt -l packages/codeupipe-go conformance/runners/go)" + - name: Vet + run: | + (cd packages/codeupipe-go && go vet ./...) + (cd conformance/runners/go && go vet ./...) + - name: Staticcheck + run: | + (cd packages/codeupipe-go && go run honnef.co/go/tools/cmd/staticcheck@2026.2.2 ./...) + (cd conformance/runners/go && go run honnef.co/go/tools/cmd/staticcheck@2026.2.2 ./...) + - name: Tests + working-directory: packages/codeupipe-go + run: go test -race ./... + - name: Example + working-directory: packages/codeupipe-go + run: go run ./examples/pricing + - name: Install the Python package, for the judge + # conformance/run.py is a codeupipe pipeline (conformance/judging/). + run: python -m pip install -e packages/codeupipe + - name: Conformance + run: python conformance/run.py go:run + - name: The oldest Go that the package supports (go.mod) + working-directory: packages/codeupipe-go + env: + GOTOOLCHAIN: go1.22.12 + run: go test -race ./... + export: name: Export against dynamic run runs-on: ubuntu-latest diff --git a/README.md b/README.md index b515675..879e4fc 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you know Unix, these are close to things you know. | Hook | A git hook: code that runs before and after a step | | State | `$?` and `PIPESTATUS` | -This repository has seven packages. +This repository has nine packages. | Package | Contents | Reference | |---|---|---| @@ -40,6 +40,7 @@ This repository has seven packages. | `codeupipe` for Rust | The primitives, 0.1.0. No dependencies | [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) | | `codeupipe-lint` for Rust | A linter for pipeline code in Rust, for the rules that the compiler does not check | [packages/codeupipe-lint-rs/README.md](packages/codeupipe-lint-rs/README.md) | | `codeupipe-compile` for Rust | Writes a pipeline as plain Rust: a struct and functions, with no codeupipe | [packages/codeupipe-compile-rs/README.md](packages/codeupipe-compile-rs/README.md) | +| `codeupipe` for Go | The eight primitives, 0.1.0. No dependencies | [packages/codeupipe-go/README.md](packages/codeupipe-go/README.md) | ## Install @@ -94,6 +95,11 @@ crates.io yet: [packages/codeupipe-rs/README.md](packages/codeupipe-rs/README.md) says how to use it from this repository. +The Go package is new too, version `0.1.0` on the same terms. No version is +tagged yet: +[packages/codeupipe-go/README.md](packages/codeupipe-go/README.md) says how to +use it from the `main` branch. + ## Example [examples/pricing/](examples/pricing/) has one class in each file. The @@ -210,6 +216,7 @@ An editable install needs pip 21.3 or later. | `packages/codeupipe-rs/` | The Rust runtime, with its tests and its examples | | `packages/codeupipe-lint-rs/` | The Rust linter, with its tests. A Cargo workspace of its own | | `packages/codeupipe-compile-rs/` | The Rust compiler, with its tests. A Cargo workspace of its own | +| `packages/codeupipe-go/` | The Go runtime, with its tests and its example. A Go module of its own | | `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runners, the benchmark | | `tests/` | The tests for the two Python packages | | `examples/pricing/` | The example, with its test in `tests/examples/` | @@ -221,7 +228,8 @@ root `package.json` is not a package: it makes the three TypeScript packages one npm workspace. The root `Cargo.toml` is not a package either: it makes the Rust runtime and its conformance runner one Cargo workspace, which builds with no download. The Rust linter needs `syn`, so it is a workspace -of its own. +of its own. The Go runtime and the Go conformance runner are two Go modules; +the runner's `go.mod` points at the runtime with a `replace`. ## Other languages @@ -229,9 +237,11 @@ The contract and the conformance cases do not depend on a language. Python is the first runtime and TypeScript is the second ([packages/codeupipe-ts/](packages/codeupipe-ts/)), with its own linter and its own compiler. Rust is the third -([packages/codeupipe-rs/](packages/codeupipe-rs/)), with its own linter; its plan is -[docs/rust-plan.md](docs/rust-plan.md). A runtime in another language must implement -the contract and add a conformance runner. +([packages/codeupipe-rs/](packages/codeupipe-rs/)), with its own linter and +its own compiler; its plan is [docs/rust-plan.md](docs/rust-plan.md). Go is +the fourth ([packages/codeupipe-go/](packages/codeupipe-go/)); its plan is +[docs/go-plan.md](docs/go-plan.md). A runtime in another language must +implement the contract and add a conformance runner. ## License diff --git a/conformance/API.md b/conformance/API.md index 83850b5..4c62c13 100644 --- a/conformance/API.md +++ b/conformance/API.md @@ -14,10 +14,10 @@ parameters, their defaults, and how they fail. | State | records what happened | | Pipeline | runs the steps in order | -Python, TypeScript and Rust implement it. Each runtime has the same contract, -with identifier casing in the style of its language, and the same +Python, TypeScript, Rust and Go implement it. Each runtime has the same +contract, with identifier casing in the style of its language, and the same conformance cases judge it. The tables below show the Python names. The -TypeScript and Rust names and differences are in "Runtimes". +TypeScript, Rust and Go names and differences are in "Runtimes". ## Principles @@ -235,16 +235,16 @@ map and fails the comparison. ## Runtimes -| | Python | TypeScript | Rust | -|---|---|---|---| -| Package | `packages/codeupipe` | `packages/codeupipe-ts` | `packages/codeupipe-rs` (crate `codeupipe`) | -| Names | snake_case | camelCase: `addFilter`, `addValve`, `toDict`, `withTrace`, `traceId`, `onError` | snake_case, as Python | -| Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | `run` is sync, and `run_with_state` for the caller's State. No async run yet | -| Absent | `None` | `null` | `None` from `get`; `Value::Null` is a JSON null | -| Label of a step | `type(step).__name__` | `step.constructor.name` | The name of the step's type, from `std::any::type_name` | -| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | `packages/codeupipe-compile-rs`: a struct with one field for each key. No conformance target yet | -| Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | None yet. The compiler refuses several of the shapes that the linters find | -| A type of your own | Any object with `copy()` | `class Order implements PipelineData`, then `new Pipeline()`, `Filter` | `impl PipelineData for Order {}`, then `Pipeline`, `Filter` | +| | Python | TypeScript | Rust | Go | +|---|---|---|---|---| +| Package | `packages/codeupipe` | `packages/codeupipe-ts` | `packages/codeupipe-rs` (crate `codeupipe`) | `packages/codeupipe-go` (package `codeupipe`) | +| Names | snake_case | camelCase: `addFilter`, `addValve`, `toDict`, `withTrace`, `traceId`, `onError` | snake_case, as Python | PascalCase: `AddFilter`, `AddValve`, `ToDict`, `WithTrace`, `TraceID`, `OnError` | +| Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | `run` is sync, and `run_with_state` for the caller's State. No async run yet | `Run(ctx, p)` is sync, and `RunWithState(ctx, p, state)` for the caller's State. Go needs no async run | +| Absent | `None` | `null` | `None` from `get`; `Value::Null` is a JSON null | `nil` from `Get`, for an absent key and for a JSON null; `Has` tells them apart | +| Label of a step | `type(step).__name__` | `step.constructor.name` | The name of the step's type, from `std::any::type_name` | The name of the step's type, from `reflect` | +| Compile | `codeupipe.compile` | `packages/codeupipe-compile-ts` | `packages/codeupipe-compile-rs`: a struct with one field for each key. No conformance target yet | None yet | +| Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | `packages/codeupipe-lint-rs` | None yet | +| A type of your own | Any object with `copy()` | `class Order implements PipelineData`, then `new Pipeline()`, `Filter` | `impl PipelineData for Order {}`, then `Pipeline`, `Filter` | A type with `Copy() *Order`, then `NewPipelineOf[*Order]()`, `FilterOf[*Order]` | Notes on TypeScript: @@ -274,8 +274,30 @@ Notes on Rust: reference, so a step can hold the pipeline that it is in (an `Rc`) and run it again. The run with the caller's State is `run_with_state(&input, &mut state)`. `state()` gives an `Rc`: the - State of the run that started last, once it has ended, as a snapshot.md](../docs/rust-plan.md) has the - plan. + State of the run that started last, once it has ended, as a snapshot. + [docs/rust-plan.md](../docs/rust-plan.md) has the plan. - **A type of your own** is a type parameter of each primitive, with `Payload` as its default, so a step for one type in a pipeline of another is a compile error. TypeScript does the same with its type check. + +Notes on Go: + +- **`ctx` comes first.** Each step method and `Run` take a + `context.Context`, as Go code that can wait does. The Pipeline gives it to + each step and does nothing else with it: a cancelled `ctx` does not stop a + run between steps. So the contract does not change. +- **Refused when added, at compile time.** A function is not a step, and the + arguments of `AddValve` are in one order only: each is a type error. A nil + step compiles, and is refused with a panic when it is added. Go has no + pointer that only reads, so a Tap, a Hook or a Predicate that writes is the + linter's rule (`CUP-I004`), as in Python and TypeScript. +- **Values.** A value is an `any` in the shapes that `encoding/json` gives: + `nil`, `bool`, `float64`, `string`, `[]any`, `map[string]any`. A number of + another Go type is kept as a `float64`. The keys have no order. +- **A State for each run.** A Pipeline can be run from many goroutines at + the same time, and each run writes into a State of its own. `State()` is + the State of the run that started last, once it has ended, as in Rust. +- **A type of your own.** Go has no default type parameter, so each + primitive has a generic form (`FilterOf[P]`) and a short name for the + Payload (`Filter` is `FilterOf[*Payload]`, the same type). + [docs/go-plan.md](../docs/go-plan.md) has the plan. diff --git a/conformance/BENCH.md b/conformance/BENCH.md index 7d3dd2e..1e770a1 100644 --- a/conformance/BENCH.md +++ b/conformance/BENCH.md @@ -15,18 +15,20 @@ two fast modes do neither, and refuse the `hook` scenario rather than silently dropping the Hook. `rust:run` is the dynamic run of the Rust crate, built with `--release`. It -records State and calls Hooks as `python:run` does. +records State and calls Hooks as `python:run` does. `go:run` is the dynamic +run of the Go package, which does the same. µs per run (best of 5 × 20,000, after 20,000 warmup runs); lower is better. -| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | rust:run | -|---|---|---|---|---|---|---| -| `micro-pass` | 10 | 4.94 | 0.824 | 0.606 | 1.92 | 0.246 | -| `valve-gated` | 10 | 5.81 | 1.11 | 0.801 | 2.25 | 0.301 | -| `taps` | 13 | 5.34 | 0.899 | 0.602 | 2.03 | 0.256 | -| `hook` | 10 | 5.88 | — | — | 2.97 | 0.247 | -| `heavy-allocation` | 50 | 16.05 | 2.84 | 1.49 | 7.01 | 4.24 | +| scenario | steps | python:run | python:static-steps | python:static-inline | python:static-observe | rust:run | go:run | +|---|---|---|---|---|---|---|---| +| `micro-pass` | 10 | 4.79 | 0.824 | 0.627 | 1.93 | 0.276 | 0.546 | +| `valve-gated` | 10 | 5.14 | 1.11 | 0.860 | 2.26 | 0.326 | 0.612 | +| `taps` | 13 | 5.03 | 0.895 | 0.626 | 2.03 | 0.293 | 0.567 | +| `hook` | 10 | 5.41 | — | — | 2.98 | 0.286 | 0.626 | +| `heavy-allocation` | 50 | 15.03 | 2.87 | 1.51 | 7.21 | 4.40 | 2.15 | Not measured (the target refuses the scenario): - python:static-steps hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast static export. Export with observe=True (top-level Hooks only), or remove them. - python:static-inline hook: UnsupportedFilterShape: .call at line 0:0 — cannot lower Hook. Hint: Hooks do not run in the fast static export. Export with observe=True (top-level Hooks only), or remove them. + diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md index 59cb9f5..2098293 100644 --- a/conformance/MATRIX.md +++ b/conformance/MATRIX.md @@ -6,7 +6,7 @@ if any target has a ❌. **Every target passes every case, and nothing is unsupported.** The targets are the Python dynamic run, async run and three compiled forms, the -TypeScript run and its compiled form, and the Rust run. +TypeScript run and its compiled form, the Rust run, and the Go run. The fast modes (`python:static-steps`, `python:static-inline`, `typescript:static`) expose no State or Hooks, so they pass those cases as @@ -16,38 +16,38 @@ the case that asks for `state` on the Pipeline object. A compiled form is a function and has no such object. See [docs/compile.md](../docs/compile.md). -| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | rust:run | -|---|---|---|---|---|---|---|---|---| -| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `state/the-caller-gives-the-state-of-a-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `state/a-run-that-starts-inside-another-run-has-its-own-state` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last` | ✅ | ✅ | ✅* | ✅* | ✅* | ✅ | ✅* | ✅ | -| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | -| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | +| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | rust:run | go:run | +|---|---|---|---|---|---|---|---|---|---| +| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `state/the-caller-gives-the-state-of-a-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `state/a-run-that-starts-inside-another-run-has-its-own-state` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `state/with-no-state-from-the-caller-the-pipeline-has-the-state-of-the-run-that-started-last` | ✅ | ✅ | ✅* | ✅* | ✅* | ✅ | ✅* | ✅ | ✅ | +| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | +| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | ✅ | ✅ | - **python:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported - **python:run_async**: 30 pass, 0 pass*, 0 fail, 0 unsupported @@ -57,5 +57,7 @@ function and has no such object. See - **typescript:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported - **typescript:static**: 11 pass, 19 pass*, 0 fail, 0 unsupported - **rust:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported +- **go:run**: 30 pass, 0 pass*, 0 fail, 0 unsupported ✅* passed everything the target can observe; it does not expose State or Hooks (the fast mode). ➖ the target cannot express the case. + diff --git a/conformance/README.md b/conformance/README.md index 6ce43fe..1282a50 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -45,8 +45,9 @@ behind function calls) that the runner times beside it, so the check does not depend on how fast the machine is. Slower than the band is a regression. Faster than the band also fails, with the instruction to record again: otherwise the next regression would hide inside the gain. There is one -baseline for each Python version, and one for the Rust run, which does not -depend on Python (the Rust runner times a reference of map writes of its own). +baseline for each Python version, and one each for the Rust and Go runs, +which do not depend on Python (each of those runners times a reference of map +writes of its own). `--record` with targets named replaces only those targets. Run it on a quiet machine: under heavy load a number can leave the band. @@ -58,6 +59,7 @@ load a number can leave the band. | `typescript:run` | `packages/codeupipe-ts` Pipeline. Node runs the `.ts` runner as it is (Node 22.18 or later). With no arguments, `run.py` leaves this target out when the machine has no such Node | | `typescript:static` | `packages/codeupipe-compile-ts`: the pipeline as plain code, written to a file and imported (no State or Hooks: ✅\*). Needs `npm ci` in that package and a built core package; `run.py` leaves it out when they are not there | | `rust:run` | `packages/codeupipe-rs` Pipeline. `cargo run --release --offline` builds the runner, which has no dependency to download. With no arguments, `run.py` leaves this target out when the machine has no `cargo` | +| `go:run` | `packages/codeupipe-go` Pipeline. `go run` builds the runner with `GOPROXY=off`: its `go.mod` points at the package with a `replace`, and it reads JSON with `encoding/json`, so it has nothing to download. With no arguments, `run.py` leaves this target out when the machine has no `go` | ## Statuses @@ -143,8 +145,9 @@ never halts the pipeline). Where it is silent, they encode Python when the Valve skips its filter. 3. **`equals` across types.** Python's `==` makes `1 == true`; JSON does not. TypeScript uses `===`, which does not compare lists or maps by content. - Rust compares `Value` by content, and a number is never equal to a + Rust and Go compare by content, and a number is never equal to a boolean. No case depends on it yet. 4. **`truthy` across languages.** An empty list or map is false in Python and true in JavaScript. Each runner uses the rule of its language; Rust - has no rule of its own and follows Python. No case depends on it yet. + and Go have no rule of their own and follow Python. No case depends on it + yet. diff --git a/conformance/bench.py b/conformance/bench.py index 6449b15..0519e32 100644 --- a/conformance/bench.py +++ b/conformance/bench.py @@ -25,8 +25,9 @@ the baseline is out of date and must be recorded again, or the next regression would hide inside the gain. A baseline is kept for each Python version (bench/baseline.json), because interpreters differ in what they make -fast. The Rust run does not depend on Python, so it has one baseline, under -"rust". --record replaces only the targets that it measured. +fast. The Rust and Go runs do not depend on Python, so each has one +baseline, under "rust" and "go". --record replaces only the targets that it +measured. """ import json @@ -47,6 +48,7 @@ "python:static-inline": ("python:static-inline", "bench:static-inline"), "python:static-observe": ("python:static-observe", "bench:static-observe"), "rust:run": ("rust:run", "bench:run"), + "go:run": ("go:run", "bench:run"), } @@ -71,8 +73,10 @@ def load_baseline(): # cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline def baseline_key(target, python): """Where the baseline of ``target`` is kept. A Python target is kept per - Python version; the Rust run does not depend on Python and has its own.""" - return "rust" if target.startswith("rust:") else python + Python version; the Rust and Go runs do not depend on Python, and each + has its own.""" + runtime = target.split(":")[0] + return runtime if runtime in ("rust", "go") else python # cup: allow-plain the speed check of the conformance suite: it times the scenarios and compares them with the baseline diff --git a/conformance/bench/baseline.json b/conformance/bench/baseline.json index a4fb61c..a7c36c8 100644 --- a/conformance/bench/baseline.json +++ b/conformance/bench/baseline.json @@ -115,6 +115,26 @@ } } }, + "go": { + "relative": { + "go:run": { + "heavy-allocation": 7.22516617444385, + "hook": 2.0373805764732418, + "micro-pass": 1.8168370390948165, + "taps": 1.898800986070889, + "valve-gated": 1.9851746428675332 + } + }, + "us_per_run": { + "go:run": { + "heavy-allocation": 2.2582312499999997, + "hook": 0.60133125, + "micro-pass": 0.5425, + "taps": 0.5583541499999999, + "valve-gated": 0.5827229 + } + } + }, "rust": { "relative": { "rust:run": { diff --git a/conformance/judging/default_targets_filter.py b/conformance/judging/default_targets_filter.py index 44588fb..885ccc6 100644 --- a/conformance/judging/default_targets_filter.py +++ b/conformance/judging/default_targets_filter.py @@ -10,15 +10,17 @@ class DefaultTargetsFilter: ({name: paths that must exist for the target}). Output keys: ``names``. A Python target always runs. A TypeScript target needs a Node that runs ``.ts`` files, and the paths of ``needs``. A Rust - target needs ``cargo``. + target needs ``cargo``, and a Go target needs ``go``. - ``node_runs_typescript`` and ``has_cargo`` are functions with no - arguments that say if the machine has such a Node, and a ``cargo``. + ``node_runs_typescript``, ``has_cargo`` and ``has_go`` are functions with + no arguments that say if the machine has such a Node, a ``cargo`` and a + ``go``. """ - def __init__(self, node_runs_typescript, has_cargo=lambda: False): + def __init__(self, node_runs_typescript, has_cargo=lambda: False, has_go=lambda: False): self.node_runs_typescript = node_runs_typescript self.has_cargo = has_cargo + self.has_go = has_go def call(self, payload: Payload) -> Payload: return payload.insert("names", self.choose(payload.get("targets"), payload.get("needs", {}))) @@ -29,6 +31,8 @@ def choose(self, targets, needs) -> list: def can_run(self, name, needs) -> bool: if name.startswith("rust:"): return self.has_cargo() + if name.startswith("go:"): + return self.has_go() if not name.startswith("typescript:"): return True return self.node_runs_typescript() and all(path.exists() for path in needs.get(name, [])) diff --git a/conformance/judging/go_probe.py b/conformance/judging/go_probe.py new file mode 100644 index 0000000..48bad1d --- /dev/null +++ b/conformance/judging/go_probe.py @@ -0,0 +1,10 @@ +"""GoProbe: what this machine has for the Go targets.""" + +import shutil + + +class GoProbe: + """Says if there is a ``go`` to build and run the Go runner.""" + + def present(self) -> bool: + return shutil.which("go") is not None diff --git a/conformance/judging/judge_pipeline.py b/conformance/judging/judge_pipeline.py index 87afcb0..8e9b63e 100644 --- a/conformance/judging/judge_pipeline.py +++ b/conformance/judging/judge_pipeline.py @@ -22,14 +22,14 @@ class JudgePipeline(Pipeline): Output keys: ``report``, ``matrix``, ``markdown``, ``exit_code``. ``launcher`` starts the runner of a target. ``node_runs_typescript`` says - if the machine can run the TypeScript targets, and ``has_cargo`` if it - can run the Rust targets. + if the machine can run the TypeScript targets, ``has_cargo`` if it can + run the Rust targets, and ``has_go`` if it can run the Go targets. """ - def __init__(self, launcher, node_runs_typescript, has_cargo=lambda: False): + def __init__(self, launcher, node_runs_typescript, has_cargo=lambda: False, has_go=lambda: False): super().__init__() self.add_filter(LoadCasesFilter()) - self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript, has_cargo)) + self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript, has_cargo, has_go)) self.add_filter(RunTargetsFilter(launcher)) self.add_filter(JudgeResultsFilter()) self.add_filter(SummarizeFilter()) diff --git a/conformance/run.py b/conformance/run.py index 126b86f..41f6949 100644 --- a/conformance/run.py +++ b/conformance/run.py @@ -4,10 +4,12 @@ python conformance/run.py python:run python:static-inline python conformance/run.py typescript:run python conformance/run.py rust:run + python conformance/run.py go:run With no arguments, the TypeScript target runs only when the machine has -Node 22.18 or later, and the Rust target only when it has cargo. A target -that is named and cannot run is a failure. +Node 22.18 or later, the Rust target only when it has cargo, and the Go +target only when it has go. A target that is named and cannot run is a +failure. A runner executes cases and reports what happened. Only this file decides pass or fail, so every target is held to exactly the same comparison. @@ -34,6 +36,7 @@ from judging.cargo_probe import CargoProbe # noqa: E402 from judging.default_targets_filter import DefaultTargetsFilter # noqa: E402 +from judging.go_probe import GoProbe # noqa: E402 from judging.judge_pipeline import JudgePipeline # noqa: E402 from judging.judge_results_filter import JudgeResultsFilter # noqa: E402 from judging.load_cases_filter import LoadCasesFilter # noqa: E402 @@ -70,6 +73,16 @@ def _rust(target): } +def _go(target): + # Built from the repository: the runner's go.mod points at the runtime + # with a replace, and GOPROXY=off makes sure that nothing is downloaded. + return { + "cmd": ["go", "run", ".", target], + "env": {"GOFLAGS": "-mod=mod", "GOPROXY": "off", "GOWORK": "off"}, + "cwd": HERE / "runners/go", + } + + TARGETS = { "python:run": _python("run"), "python:run_async": _python("run_async"), @@ -79,6 +92,7 @@ def _rust(target): "typescript:run": _typescript("run"), "typescript:static": _typescript("static"), "rust:run": _rust("run"), + "go:run": _go("run"), } # The static target runs the compile package. Its dependencies must be @@ -103,11 +117,17 @@ def _has_cargo(): return CargoProbe().present() +def _has_go(): + """Whether there is a go to build and run the Go runner.""" + return GoProbe().present() + + def default_targets(): """Every target that this machine can run. TypeScript needs Node, the - static TypeScript target needs its package to be installed, and Rust - needs cargo.""" - return DefaultTargetsFilter(lambda: _node_runs_typescript(), lambda: _has_cargo()).choose(TARGETS, _TYPESCRIPT_NEEDS) + static TypeScript target needs its package to be installed, Rust needs + cargo, and Go needs go.""" + return DefaultTargetsFilter(lambda: _node_runs_typescript(), lambda: _has_cargo(), + lambda: _has_go()).choose(TARGETS, _TYPESCRIPT_NEEDS) def load_cases(): @@ -129,7 +149,8 @@ def judge(case, result): def main(argv): - pipeline = JudgePipeline(ProcessLauncher(), lambda: _node_runs_typescript(), lambda: _has_cargo()) + pipeline = JudgePipeline(ProcessLauncher(), lambda: _node_runs_typescript(), lambda: _has_cargo(), + lambda: _has_go()) out = pipeline.run(Payload({"cases_dir": CASES, "results_dir": RESULTS, "targets": TARGETS, "needs": _TYPESCRIPT_NEEDS, "names": list(argv)})) print(out.get("markdown")) diff --git a/conformance/runners/go/bench.go b/conformance/runners/go/bench.go new file mode 100644 index 0000000..b1d584a --- /dev/null +++ b/conformance/runners/go/bench.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "runtime" + "sort" + "time" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// The bench mode times the scenarios as the Python and Rust runners do, so +// that conformance/bench.py can put the numbers side by side. + +const ( + warmup = 20_000 + runs = 20_000 + repeats = 5 +) + +var referenceKeys = [10]string{"k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8", "k9"} + +//go:noinline +func referenceWrite(data map[string]float64, key string) { + data[key]++ +} + +// reference is the reference workload: ten map writes behind ten function +// calls, with no framework. A pipeline's time is reported relative to it. +func reference() map[string]float64 { + data := map[string]float64{"a": 1} + for _, key := range referenceKeys { + referenceWrite(data, key) + } + return data +} + +func timed(f func()) time.Duration { + started := time.Now() + for i := 0; i < runs; i++ { + f() + } + return time.Since(started) +} + +// measure gives µs per run, the best of repeats × runs, and the time +// relative to the reference workload. Each repeat times the reference and +// then f, back to back, and the median of the ratios is given. +func measure(f func()) (float64, float64) { + // runtime.KeepAlive keeps each result, so that the compiler cannot drop the work. + referenceRun := func() { runtime.KeepAlive(reference()) } + for i := 0; i < warmup; i++ { + f() + referenceRun() + } + best := time.Duration(1<<63 - 1) + ratios := make([]float64, 0, repeats) + for i := 0; i < repeats; i++ { + ref := timed(referenceRun) + elapsed := timed(f) + ratios = append(ratios, elapsed.Seconds()/ref.Seconds()) + if elapsed < best { + best = elapsed + } + } + sort.Float64s(ratios) + return best.Seconds() / runs * 1e6, ratios[repeats/2] +} + +// bench times one scenario. pipeline is built from its steps; input is its +// input. +func bench(id any, pipeline *codeupipe.Pipeline, input *codeupipe.Payload) map[string]any { + ctx := context.Background() + output, err := pipeline.Run(ctx, input) + if err != nil { + return map[string]any{"id": id, "unsupported": "the scenario fails: " + err.Error()} + } + usPerRun, relative := measure(func() { + out, _ := pipeline.Run(ctx, input) + runtime.KeepAlive(out) + }) + return map[string]any{ + "id": id, + "us_per_run": usPerRun, + "relative": relative, + "output": output.ToDict(), + } +} diff --git a/conformance/runners/go/cup_ops/add_filter.go b/conformance/runners/go/cup_ops/add_filter.go new file mode 100644 index 0000000..ff3a672 --- /dev/null +++ b/conformance/runners/go/cup_ops/add_filter.go @@ -0,0 +1,28 @@ +package cup_ops + +import ( + "context" + "fmt" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// AddFilter inserts key = get(key, 0) + by. A value that is there and is not +// a number is an error, as in Python. +type AddFilter struct { + Key string + By float64 +} + +func (f AddFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + n := 0.0 + if payload.Has(f.Key) { + value, ok := payload.Get(f.Key).(float64) + if !ok { + return fmt.Errorf("'%s' is not a number: %v", f.Key, payload.Get(f.Key)) + } + n = value + } + payload.Insert(f.Key, n+f.By) + return nil +} diff --git a/conformance/runners/go/cup_ops/append_filter.go b/conformance/runners/go/cup_ops/append_filter.go new file mode 100644 index 0000000..8ab7f3c --- /dev/null +++ b/conformance/runners/go/cup_ops/append_filter.go @@ -0,0 +1,28 @@ +package cup_ops + +import ( + "context" + "fmt" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// AppendFilter inserts key = a copy of get(key), or of an empty list, with +// value added. The list that is there is not changed. +type AppendFilter struct { + Key string + Value any +} + +func (f AppendFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + var list []any + switch items := payload.Get(f.Key).(type) { + case nil: + case []any: + list = append(list, items...) + default: + return fmt.Errorf("'%s' is not a list: %v", f.Key, items) + } + payload.Insert(f.Key, append(list, f.Value)) + return nil +} diff --git a/conformance/runners/go/cup_ops/copy_filter.go b/conformance/runners/go/cup_ops/copy_filter.go new file mode 100644 index 0000000..2e533c7 --- /dev/null +++ b/conformance/runners/go/cup_ops/copy_filter.go @@ -0,0 +1,18 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// CopyFilter inserts to = get(from), nil if missing. +type CopyFilter struct { + From string + To string +} + +func (f CopyFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Insert(f.To, payload.Get(f.From)) + return nil +} diff --git a/conformance/runners/go/cup_ops/equals_predicate.go b/conformance/runners/go/cup_ops/equals_predicate.go new file mode 100644 index 0000000..6fe5901 --- /dev/null +++ b/conformance/runners/go/cup_ops/equals_predicate.go @@ -0,0 +1,19 @@ +package cup_ops + +import ( + "context" + "reflect" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// EqualsPredicate is true when get(key) == value, by content. A missing key +// is nil. A number is never equal to a bool: they are values of two types. +type EqualsPredicate struct { + Key string + Value any +} + +func (q EqualsPredicate) Test(ctx context.Context, payload *codeupipe.Payload) bool { + return reflect.DeepEqual(payload.Get(q.Key), q.Value) +} diff --git a/conformance/runners/go/cup_ops/fail_filter.go b/conformance/runners/go/cup_ops/fail_filter.go new file mode 100644 index 0000000..b6121be --- /dev/null +++ b/conformance/runners/go/cup_ops/fail_filter.go @@ -0,0 +1,17 @@ +package cup_ops + +import ( + "context" + "errors" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// FailFilter returns an error that carries message. +type FailFilter struct { + Message string +} + +func (f FailFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + return errors.New(f.Message) +} diff --git a/conformance/runners/go/cup_ops/fail_tap.go b/conformance/runners/go/cup_ops/fail_tap.go new file mode 100644 index 0000000..88cf345 --- /dev/null +++ b/conformance/runners/go/cup_ops/fail_tap.go @@ -0,0 +1,17 @@ +package cup_ops + +import ( + "context" + "errors" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// FailTap returns an error that carries message. +type FailTap struct { + Message string +} + +func (t FailTap) Observe(ctx context.Context, payload *codeupipe.Payload) error { + return errors.New(t.Message) +} diff --git a/conformance/runners/go/cup_ops/has_filter.go b/conformance/runners/go/cup_ops/has_filter.go new file mode 100644 index 0000000..65353ab --- /dev/null +++ b/conformance/runners/go/cup_ops/has_filter.go @@ -0,0 +1,18 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// HasFilter inserts to = has(key). +type HasFilter struct { + Key string + To string +} + +func (f HasFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Insert(f.To, payload.Has(f.Key)) + return nil +} diff --git a/conformance/runners/go/cup_ops/merge_filter.go b/conformance/runners/go/cup_ops/merge_filter.go new file mode 100644 index 0000000..6a2151d --- /dev/null +++ b/conformance/runners/go/cup_ops/merge_filter.go @@ -0,0 +1,17 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// MergeFilter merges a Payload of value into the Payload. +type MergeFilter struct { + Value map[string]any +} + +func (f MergeFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Merge(codeupipe.NewPayload(f.Value)) + return nil +} diff --git a/conformance/runners/go/cup_ops/mutate_filter.go b/conformance/runners/go/cup_ops/mutate_filter.go new file mode 100644 index 0000000..42885c9 --- /dev/null +++ b/conformance/runners/go/cup_ops/mutate_filter.go @@ -0,0 +1,20 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// MutateFilter inserts key = value as a statement of its own. A Go Filter +// always changes the Payload in place, so this is the same as SetFilter; the +// case is kept so that every runtime runs it. +type MutateFilter struct { + Key string + Value any +} + +func (f MutateFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Insert(f.Key, f.Value) + return nil +} diff --git a/conformance/runners/go/cup_ops/nested_pipeline.go b/conformance/runners/go/cup_ops/nested_pipeline.go new file mode 100644 index 0000000..7ba96ba --- /dev/null +++ b/conformance/runners/go/cup_ops/nested_pipeline.go @@ -0,0 +1,9 @@ +package cup_ops + +import "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" + +// NestedPipeline is a named pipeline type, for a pipeline that is a step of +// another. The runner adds the steps of the case after it is made. +type NestedPipeline struct{} + +func (NestedPipeline) AddSteps(pipeline *codeupipe.Pipeline) {} diff --git a/conformance/runners/go/cup_ops/noop_hook.go b/conformance/runners/go/cup_ops/noop_hook.go new file mode 100644 index 0000000..59078e8 --- /dev/null +++ b/conformance/runners/go/cup_ops/noop_hook.go @@ -0,0 +1,8 @@ +package cup_ops + +import "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" + +// NoopHook is a Hook that does nothing: the cost of having one. +type NoopHook struct { + codeupipe.HookBase +} diff --git a/conformance/runners/go/cup_ops/noop_tap.go b/conformance/runners/go/cup_ops/noop_tap.go new file mode 100644 index 0000000..d577b39 --- /dev/null +++ b/conformance/runners/go/cup_ops/noop_tap.go @@ -0,0 +1,14 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// NoopTap does nothing. +type NoopTap struct{} + +func (NoopTap) Observe(ctx context.Context, payload *codeupipe.Payload) error { + return nil +} diff --git a/conformance/runners/go/cup_ops/record_hook.go b/conformance/runners/go/cup_ops/record_hook.go new file mode 100644 index 0000000..2222f17 --- /dev/null +++ b/conformance/runners/go/cup_ops/record_hook.go @@ -0,0 +1,25 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// RecordHook records "before:", "after:" and "error:". +// Events of the pipeline itself have an empty step. +type RecordHook struct { + Events *[]string +} + +func (h RecordHook) Before(ctx context.Context, step string, payload *codeupipe.Payload) { + *h.Events = append(*h.Events, "before:"+step) +} + +func (h RecordHook) After(ctx context.Context, step string, payload *codeupipe.Payload) { + *h.Events = append(*h.Events, "after:"+step) +} + +func (h RecordHook) OnError(ctx context.Context, step string, err error, payload *codeupipe.Payload) { + *h.Events = append(*h.Events, "error:"+step) +} diff --git a/conformance/runners/go/cup_ops/record_tap.go b/conformance/runners/go/cup_ops/record_tap.go new file mode 100644 index 0000000..2211378 --- /dev/null +++ b/conformance/runners/go/cup_ops/record_tap.go @@ -0,0 +1,17 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// RecordTap adds get(key) to the tap log of the runner. +type RecordTap struct { + Key string +} + +func (t RecordTap) Observe(ctx context.Context, payload *codeupipe.Payload) error { + Record(payload.Get(t.Key)) + return nil +} diff --git a/conformance/runners/go/cup_ops/remove_filter.go b/conformance/runners/go/cup_ops/remove_filter.go new file mode 100644 index 0000000..ced8bc7 --- /dev/null +++ b/conformance/runners/go/cup_ops/remove_filter.go @@ -0,0 +1,17 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// RemoveFilter removes key. +type RemoveFilter struct { + Key string +} + +func (f RemoveFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Remove(f.Key) + return nil +} diff --git a/conformance/runners/go/cup_ops/run_again_filter.go b/conformance/runners/go/cup_ops/run_again_filter.go new file mode 100644 index 0000000..73144d0 --- /dev/null +++ b/conformance/runners/go/cup_ops/run_again_filter.go @@ -0,0 +1,22 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// RunAgainFilter runs the pipeline that it is in one more time, with Input +// and a new State, when get("inner") is not true. How a pipeline is run +// again depends on the target, so the runner gives that as a function +// (SetRunAgain). Then the payload goes on with no change. +type RunAgainFilter struct { + Input map[string]any +} + +func (f RunAgainFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + if payload.Get("inner") == true { + return nil + } + return RunAgain(ctx, f.Input) +} diff --git a/conformance/runners/go/cup_ops/set_filter.go b/conformance/runners/go/cup_ops/set_filter.go new file mode 100644 index 0000000..ece317a --- /dev/null +++ b/conformance/runners/go/cup_ops/set_filter.go @@ -0,0 +1,18 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// SetFilter inserts key = value. +type SetFilter struct { + Key string + Value any +} + +func (f SetFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Insert(f.Key, f.Value) + return nil +} diff --git a/conformance/runners/go/cup_ops/sink.go b/conformance/runners/go/cup_ops/sink.go new file mode 100644 index 0000000..2e7940e --- /dev/null +++ b/conformance/runners/go/cup_ops/sink.go @@ -0,0 +1,59 @@ +// Package cup_ops has the steps of the cases: one type for each step, with +// the names that the TypeScript and Rust runners use, so that the labels are +// the same. +package cup_ops + +import ( + "context" + "errors" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// Where the steps of the runner record what they see, and the function that +// runs the pipeline of a case again. The runner plays one case at a time, in +// one goroutine, and empties these before each case. +var ( + taps []any + runAgain func(ctx context.Context, input map[string]any) error + innerStates []*codeupipe.State +) + +// Record adds a value to the tap log. +func Record(value any) { + taps = append(taps, value) +} + +// TakeTaps gives what was recorded since the last call, and empties the log. +func TakeTaps() []any { + recorded := append([]any{}, taps...) + taps = nil + return recorded +} + +// SetRunAgain sets how the RunAgainFilter of this case runs its pipeline +// again. nil removes it. +func SetRunAgain(run func(ctx context.Context, input map[string]any) error) { + runAgain = run +} + +// RunAgain runs the pipeline of the case again, with input and a new State. +func RunAgain(ctx context.Context, input map[string]any) error { + if runAgain == nil { + return errors.New("no pipeline to run again") + } + return runAgain(ctx, input) +} + +// KeepInnerState keeps the State of a run that a RunAgainFilter started. +func KeepInnerState(state *codeupipe.State) { + innerStates = append(innerStates, state) +} + +// TakeInnerStates gives the States kept since the last call, and empties +// the list. +func TakeInnerStates() []*codeupipe.State { + kept := innerStates + innerStates = nil + return kept +} diff --git a/conformance/runners/go/cup_ops/truthy_predicate.go b/conformance/runners/go/cup_ops/truthy_predicate.go new file mode 100644 index 0000000..92105c8 --- /dev/null +++ b/conformance/runners/go/cup_ops/truthy_predicate.go @@ -0,0 +1,31 @@ +package cup_ops + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// TruthyPredicate is true when get(key) is truthy, as in Python: nil, false, +// 0, "" and an empty list or map are false. A missing key is false. +type TruthyPredicate struct { + Key string +} + +func (q TruthyPredicate) Test(ctx context.Context, payload *codeupipe.Payload) bool { + switch value := payload.Get(q.Key).(type) { + case nil: + return false + case bool: + return value + case float64: + return value != 0 + case string: + return value != "" + case []any: + return len(value) > 0 + case map[string]any: + return len(value) > 0 + } + return true +} diff --git a/conformance/runners/go/go.mod b/conformance/runners/go/go.mod new file mode 100644 index 0000000..d82df49 --- /dev/null +++ b/conformance/runners/go/go.mod @@ -0,0 +1,10 @@ +// The Go conformance runner: go run . < cases.json > results.json. +// It has no dependency to download: the runtime is the package of this +// repository, by the replace below. +module github.com/orchestrate-solutions/codeupipe/conformance/runners/go + +go 1.22 + +require github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go v0.0.0 + +replace github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go => ../../../packages/codeupipe-go diff --git a/conformance/runners/go/main.go b/conformance/runners/go/main.go new file mode 100644 index 0000000..0b1c19e --- /dev/null +++ b/conformance/runners/go/main.go @@ -0,0 +1,435 @@ +// The Go conformance runner: it runs the cases and reports what happened. +// +// go run . < cases.json > results.json +// +// Targets: +// +// run the Pipeline of packages/codeupipe-go +// bench:run the same, timed on the scenarios of conformance/bench +// +// The runner never judges; conformance/run.py compares the results of every +// target with the same expectations. +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/orchestrate-solutions/codeupipe/conformance/runners/go/cup_ops" + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// unsupported is a case that this runner cannot express. It is reported, +// never judged. +type unsupported string + +func (u unsupported) Error() string { return string(u) } + +func field(spec map[string]any, name string) (any, error) { + value, ok := spec[name] + if !ok { + return nil, unsupported(fmt.Sprintf("the step has no '%s'", name)) + } + return value, nil +} + +func text(spec map[string]any, name string) (string, error) { + value, err := field(spec, name) + if err != nil { + return "", err + } + s, ok := value.(string) + if !ok { + return "", unsupported(fmt.Sprintf("'%s' is not a string", name)) + } + return s, nil +} + +func number(spec map[string]any, name string) (float64, error) { + value, err := field(spec, name) + if err != nil { + return 0, err + } + n, ok := value.(float64) + if !ok { + return 0, unsupported(fmt.Sprintf("'%s' is not a number", name)) + } + return n, nil +} + +func object(spec map[string]any, name string) (map[string]any, error) { + value, err := field(spec, name) + if err != nil { + return nil, err + } + m, ok := value.(map[string]any) + if !ok { + return nil, unsupported(fmt.Sprintf("'%s' is not an object", name)) + } + return m, nil +} + +// filterOf gives the Filter of a step of the case. +func filterOf(spec map[string]any) (codeupipe.Filter, error) { + op, err := text(spec, "op") + if err != nil { + return nil, err + } + var key, other string + var problem error + read := func(name string) string { + s, err := text(spec, name) + if err != nil && problem == nil { + problem = err + } + return s + } + switch op { + case "set", "mutate", "append": + key = read("key") + value, err := field(spec, "value") + if err != nil { + return nil, err + } + if problem != nil { + return nil, problem + } + switch op { + case "set": + return cup_ops.SetFilter{Key: key, Value: value}, nil + case "mutate": + return cup_ops.MutateFilter{Key: key, Value: value}, nil + } + return cup_ops.AppendFilter{Key: key, Value: value}, nil + case "copy": + key, other = read("from"), read("to") + return cup_ops.CopyFilter{From: key, To: other}, problem + case "add": + key = read("key") + by, err := number(spec, "by") + if err != nil { + return nil, err + } + return cup_ops.AddFilter{Key: key, By: by}, problem + case "remove": + key = read("key") + return cup_ops.RemoveFilter{Key: key}, problem + case "merge": + value, err := object(spec, "value") + if err != nil { + return nil, err + } + return cup_ops.MergeFilter{Value: value}, nil + case "has": + key, other = read("key"), read("to") + return cup_ops.HasFilter{Key: key, To: other}, problem + case "fail": + return cup_ops.FailFilter{Message: read("message")}, problem + case "run_again": + input, err := object(spec, "input") + if err != nil { + return nil, err + } + return cup_ops.RunAgainFilter{Input: input}, nil + } + return nil, unsupported(fmt.Sprintf("unknown filter op '%s'", op)) +} + +// tapOf gives the Tap of a step of the case. +func tapOf(spec map[string]any) (codeupipe.Tap, error) { + op, err := text(spec, "op") + if err != nil { + return nil, err + } + switch op { + case "record": + key, err := text(spec, "key") + return cup_ops.RecordTap{Key: key}, err + case "fail": + message, err := text(spec, "message") + return cup_ops.FailTap{Message: message}, err + case "noop": + return cup_ops.NoopTap{}, nil + } + return nil, unsupported(fmt.Sprintf("unknown tap op '%s'", op)) +} + +// predicateOf gives the Predicate of the "when" of a valve. +func predicateOf(when map[string]any) (codeupipe.Predicate, error) { + op, err := text(when, "op") + if err != nil { + return nil, err + } + key, err := text(when, "key") + if err != nil { + return nil, err + } + switch op { + case "equals": + value, err := field(when, "value") + return cup_ops.EqualsPredicate{Key: key, Value: value}, err + case "truthy": + return cup_ops.TruthyPredicate{Key: key}, nil + } + return nil, unsupported(fmt.Sprintf("unknown valve op '%s'", op)) +} + +func stepsOf(spec map[string]any) ([]any, error) { + value, err := field(spec, "steps") + if err != nil { + return nil, err + } + steps, ok := value.([]any) + if !ok { + return nil, unsupported("'steps' is not a list") + } + return steps, nil +} + +// build adds the steps of the case to pipeline. +func build(steps []any, pipeline *codeupipe.Pipeline) error { + for _, item := range steps { + spec, ok := item.(map[string]any) + if !ok { + return unsupported("a step is not an object") + } + kind, err := text(spec, "kind") + if err != nil { + return err + } + switch kind { + case "filter": + filter, err := filterOf(spec) + if err != nil { + return err + } + pipeline.AddFilter(filter) + case "tap": + tap, err := tapOf(spec) + if err != nil { + return err + } + pipeline.AddTap(tap) + case "valve": + when, err := object(spec, "when") + if err != nil { + return err + } + predicate, err := predicateOf(when) + if err != nil { + return err + } + gated, err := object(spec, "filter") + if err != nil { + return err + } + filter, err := filterOf(gated) + if err != nil { + return err + } + pipeline.AddValve(predicate, filter) + case "pipeline": + inner := codeupipe.NewNamedPipeline(cup_ops.NestedPipeline{}) + steps, err := stepsOf(spec) + if err != nil { + return err + } + if err := build(steps, inner); err != nil { + return err + } + pipeline.AddPipeline(inner) + default: + return unsupported(fmt.Sprintf("unknown step kind '%s'", kind)) + } + } + return nil +} + +// namesByLabel maps each label to the step name of the case. The cases name +// their steps, and the runtime reports labels (conformance/API.md). The +// steps are added in the order of the case, so the two lists are side by +// side. A label that the runtime reports and that is not in this map stays +// as it is, and fails the comparison. +func namesByLabel(pipeline *codeupipe.Pipeline, steps []any) func(string) string { + names := make(map[string]string) + for i, label := range pipeline.Labels() { + spec, _ := steps[i].(map[string]any) + name, _ := spec["name"].(string) + names[label] = name + } + return func(label string) string { + if name, ok := names[label]; ok { + return name + } + return label + } +} + +func named(labels []string, name func(string) string) []any { + out := make([]any, len(labels)) + for i, label := range labels { + out[i] = name(label) + } + return out +} + +func report(state *codeupipe.State, name func(string) string) map[string]any { + var errors []string + for _, failure := range state.Errors() { + errors = append(errors, failure.Label) + } + return map[string]any{ + "executed": named(state.Executed(), name), + "skipped": named(state.Skipped(), name), + "errors": named(errors, name), + } +} + +func runCase(spec map[string]any) (map[string]any, error) { + ctx := context.Background() + steps, err := stepsOf(spec) + if err != nil { + return nil, err + } + pipeline := codeupipe.NewPipeline() + if err := build(steps, pipeline); err != nil { + return nil, err + } + name := namesByLabel(pipeline, steps) + hooks := spec["hooks"] == true + var events []string + if hooks { + pipeline.UseHook(cup_ops.RecordHook{Events: &events}) + } + + // A run_again step runs this pipeline again, as the TypeScript and Rust + // runners do. + cup_ops.SetRunAgain(func(ctx context.Context, input map[string]any) error { + state := codeupipe.NewState() + _, err := pipeline.RunWithState(ctx, codeupipe.NewPayload(input), state) + cup_ops.KeepInnerState(state) + return err + }) + defer cup_ops.SetRunAgain(nil) + + data, err := object(spec, "input") + if err != nil { + return nil, err + } + input := codeupipe.NewPayload(data) + result := map[string]any{"id": spec["id"]} + // With "given_state", the caller gives the State of the run. If not, the + // run makes one. + var given *codeupipe.State + var output *codeupipe.Payload + if spec["given_state"] == true { + given = codeupipe.NewState() + output, err = pipeline.RunWithState(ctx, input, given) + } else { + output, err = pipeline.Run(ctx, input) + } + if err != nil { + result["error"] = err.Error() + } else { + result["output"] = output.ToDict() + } + last := pipeline.State() + if given == nil { + given = last + } + result["state"] = report(given, name) + result["pipeline_state"] = report(last, name) + inner := []any{} + for _, state := range cup_ops.TakeInnerStates() { + inner = append(inner, report(state, name)) + } + result["inner_states"] = inner + result["taps"] = cup_ops.TakeTaps() + if hooks { + reported := make([]any, len(events)) + for i, event := range events { + kind, label, _ := strings.Cut(event, ":") + if label != "" { + label = name(label) + } + reported[i] = kind + ":" + label + } + result["hooks"] = reported + } + result["input_after"] = input.ToDict() + return result, nil +} + +func benchCase(spec map[string]any) (map[string]any, error) { + steps, err := stepsOf(spec) + if err != nil { + return nil, err + } + pipeline := codeupipe.NewPipeline() + if err := build(steps, pipeline); err != nil { + return nil, err + } + if spec["hooks"] == "noop" { + pipeline.UseHook(cup_ops.NoopHook{}) + } + data, err := object(spec, "input") + if err != nil { + return nil, err + } + return bench(spec["id"], pipeline, codeupipe.NewPayload(data)), nil +} + +func main() { + target := "" + if len(os.Args) > 1 { + target = os.Args[1] + } + var play func(map[string]any) (map[string]any, error) + switch target { + case "run": + play = runCase + case "bench:run": + play = benchCase + default: + fmt.Fprintf(os.Stderr, "unknown target '%s'. The targets are: run, bench:run\n", target) + os.Exit(2) + } + text, err := io.ReadAll(os.Stdin) + if err != nil { + fmt.Fprintln(os.Stderr, "cannot read the cases:", err) + os.Exit(2) + } + var cases []any + if err := json.Unmarshal(text, &cases); err != nil { + fmt.Fprintln(os.Stderr, "cannot read the cases:", err) + os.Exit(2) + } + + results := make([]any, 0, len(cases)) + for _, item := range cases { + cup_ops.TakeTaps() + spec, ok := item.(map[string]any) + if !ok { + fmt.Fprintln(os.Stderr, "a case is not an object") + os.Exit(2) + } + result, err := play(spec) + if reason, ok := err.(unsupported); ok { + result = map[string]any{"id": spec["id"], "unsupported": string(reason)} + } else if err != nil { + fmt.Fprintln(os.Stderr, "cannot play a case:", err) + os.Exit(2) + } + results = append(results, result) + } + out := json.NewEncoder(os.Stdout) + out.SetEscapeHTML(false) + if err := out.Encode(results); err != nil { + fmt.Fprintln(os.Stderr, "cannot write the results:", err) + os.Exit(2) + } +} diff --git a/docs/go-plan.md b/docs/go-plan.md new file mode 100644 index 0000000..de316fb --- /dev/null +++ b/docs/go-plan.md @@ -0,0 +1,442 @@ +# Plan: codeupipe for Go + +This is a plan, not a reference. It says how the fourth runtime is added: a +Go runtime with the same contract ([conformance/API.md](../conformance/API.md)), +judged by the same conformance cases, in the same layout as TypeScript and +Rust. + +**How it is built: from the contract, tests first.** Each behaviour starts as +a failing test that comes from a rule of the contract or from a test of the +TypeScript package. **When in doubt, do what the TypeScript package does**: it +is the reference for every question that the contract does not answer. Where +Rust already answered a question that Go also raises, the Rust answer is the +start ([rust-plan.md](rust-plan.md)). + +The Go runtime in the old repository (`runtimes/go`) is not the start. Its +API is from before version 2: steps named by hand, a Payload that does not +change, no Predicate, and a dependency (`msgpack`). It can be read for ideas. +No code is copied from it. + +## What Go gives for free + +- **A JSON value.** `encoding/json` reads JSON into `any`: `nil`, `bool`, + `float64`, `string`, `[]any`, `map[string]any`. That is the contract's value + (D1). Rust had to write its own `Value` and its own JSON reader. Go needs + neither, in the runtime or in the conformance runner. +- **A reader and a writer of Go.** `go/parser`, `go/ast`, `go/types` and + `go/format` are in the standard library. The compiler (phase 3) and the + linter (phase 4) need no dependency. Rust needs `syn`, TypeScript needs + `typescript`. +- **Labels.** An interface value keeps the type of the step, so + `reflect.TypeOf` gives its name when the step is added. The Rust runner + needed a trick (`UseFilter`) to keep the type of a step it picks at run + time. The Go runner does not. + +## Versions + +The Go module starts at **0.1.0**, as TypeScript and Rust did. A version +below 1.0 is not stable yet. + +Go has no registry to publish to: a tag on a public repository is a release. +For a module in a subdirectory, the tag has the directory in its name: +`packages/codeupipe-go/v0.1.0`. **No tag is made until it is decided**, as each +Rust crate has `publish = false`. Until then a project uses the main branch: + +```bash +go get github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go@main +``` + +## Layout + +| Path | Module or package | Contents | Phase | +|---|---|---|---| +| `packages/codeupipe-go/` | package `codeupipe` | The eight primitives. No dependencies | 1 | +| `conformance/runners/go/` | (a command, not published) | The conformance runner, with its `cup_ops` steps | 2 | +| `packages/codeupipe-compile-go/` | `codeupipe-compile` | Writes a pipeline as plain Go | 3 | +| `packages/codeupipe-lint-go/` | `codeupipe-lint` | The linter for the rules that the types do not check | 4 | + +One primitive is in each file, and the file name is the type name in +snake_case, as in Python and Rust: `DiscountFilter` is in +`discount_filter.go`. Go keeps a test beside its source, so the repository's +rule "tests mirror source" is the language's own: `payload.go` and +`payload_test.go`. + +### The core package + +``` +packages/codeupipe-go/ + go.mod module github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go + README.md LICENSE NOTICE + payload.go Payload + filter.go FilterOf[P], and Filter for a Payload + predicate.go PredicateOf[P], and Predicate + tap.go TapOf[P], and Tap + hook.go HookOf[P], and Hook + state.go State + valve.go valve (not a public name) + pipeline.go PipelineOf[P], and Pipeline + pipeline_data.go PipelineData[P]: the one thing a type of your own needs + named_pipeline.go NamedPipeline (see D7) + step_label.go the label from the type name + *_test.go one beside each file that it tests + examples/pricing/ + main.go discount_filter.go mark_bulk_filter.go at_least_predicate.go + audit_tap.go print_steps_hook.go pricing_pipeline.go +``` + +## Decisions + +These are the questions that Go raises. **They are settled as recommended** +(2026-10-29), with `ctx` on every step (D4). + +### D1. Values + +**The question.** Values are JSON-shaped, and the core has no dependencies. + +**Recommendation.** A value is an `any` in the shapes that `encoding/json` +gives: `nil`, `bool`, `float64`, `string`, `[]any`, `map[string]any`. There is +no `Value` type of codeupipe. One number type, `float64`, as JSON, JavaScript +and the Rust runtime have, and as the judge compares (`1 == 1.0`). + +`Insert` changes Go's other number types (`int`, `int64`, `float32` and the +others) to `float64`. Without that, `payload.Insert("qty", 12)` would be read +back by `payload.Get("qty").(float64)` as "not a number", with no error. The +change is a type switch on the value, so it costs almost nothing. It is done at +the top of the value only: a list or a map is kept as it is given. The runtime +does not check other shapes (a struct, a `[]int`). The linter can (phase 4). + +The keys have no order, as a Go map has none. Rust kept the order in which +keys were added, for people; the judge does not compare order. `fmt` prints a +map with its keys sorted, so a printed Payload is still the same each time. + +### D2. `get` with a default + +**The question.** The contract has `get(k, default)`. Go has no default +arguments and no overloads. + +**Recommendation.** `Get(key string) any`, which gives `nil` when the key is +absent. `nil` is Go's null, and the contract says that absent is the +language's null. `Has` tells a key with a null value from an absent key. + +A default is Go's own way, a type assertion that gives a zero value: + +```go +price, _ := payload.Get("price").(float64) // 0 when absent, null or not a number +qty, ok := payload.Get("qty").(float64) +if !ok { + qty = 1 +} +``` + +This is what Rust did with `.unwrap_or(...)`: the language's way, not a second +name. The other choice is `Get(key string, fallback ...any) any`, which is +nearer to Python and TypeScript, but lets `Get("a", 1, 2)` compile. + +### D3. A write, and what a step gets + +**The question.** The contract says that a write changes the Payload and +returns it. + +**Recommendation.** As TypeScript and Rust: a Filter changes the Payload in +place, and a write returns it, so writes chain. + +```go +func (f DiscountFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + price, _ := payload.Get("price").(float64) + payload.Insert("price", math.Round(price*(1-f.Rate)*100)/100) + return nil +} +``` + +When a Filter fails, `OnError` gets the same Payload, with the writes that the +Filter made before it failed. That is what TypeScript does. + +A Tap, a Predicate and a Hook also get `*Payload`. Go has no pointer that only +reads, so **a write outside a Filter is not a type error, as it is in Rust**. +It is the linter's rule `CUP-I004`, as in Python and TypeScript. An interface +that only reads was looked at and is not recommended: it cannot cover a type +of your own (D9), and each read would be a call through an interface. + +### D4. `context.Context` + +**The question.** In Go, a function that can wait (on the network, a file, a +database) takes a `context.Context` first, for cancellation and deadlines. The +contract has no cancellation. **This is the decision that changes how every +step looks**, so it is made now: to add it later changes every step. + +**Recommendation.** Every method of a step takes `ctx` first, and `Run` takes +it too: `Call(ctx, payload)`, `Test(ctx, payload)`, `Observe(ctx, payload)`, +`Before(ctx, step, payload)`, `Run(ctx, input)`. **The Pipeline gives `ctx` to +each step and does nothing else with it**: it does not stop between steps when +`ctx` is cancelled. So the contract does not change. A step that calls +`http.NewRequestWithContext` or `db.QueryContext` gives it the caller's `ctx`, +which is what a Go reviewer looks for. + +Stopping a run between steps when `ctx` is cancelled would be a new rule. It +enters the contract with a conformance case, if a real pipeline needs it. + +The other choice is no `ctx` in 0.1.0 (principle 5: nothing that is not in the +contract). Then a step that needs one has no way to get the caller's, and uses +`context.Background()`. + +### D5. Errors and panics + +**Recommendation.** As Rust. A step returns `error`. `Run` returns that same +error, so `errors.Is` and `errors.As` work on it. `OnError` gets it, and +`State.Errors` keeps the label and the error. + +**A panic is not a failure of a step** and is not caught: a panicking Tap +stops the run. The README says so. Catching a panic with `recover` can be added +later with no change to the API. + +### D6. Labels + +**Recommendation.** `reflect.TypeOf` of the step when it is added. For a +pointer, the type that it points to. Type arguments are dropped +(`Wrapper[pkg.Inner]` gives `Wrapper`). A type with no name (a struct literal +that embeds a step) gives `anonymous`, as in TypeScript. Checked on Go 1.25 +with a `go 1.22` module: + +| Step | `reflect` name | Label | +|---|---|---| +| `DiscountFilter{}` | `DiscountFilter` | `DiscountFilter` | +| `&DiscountFilter{}` | (pointer) | `DiscountFilter` | +| `Wrapper[DiscountFilter]{}` | `Wrapper[main.DiscountFilter]` | `Wrapper` | +| `struct{ DiscountFilter }{}` | `` | `anonymous` | + +What the contract refuses with a `TypeError` is a compile error in Go: a value +with no `Call` does not implement the Filter interface, and +`AddValve(filter, predicate)` does not compile, since a Filter has no `Test`. + +A plain function is not a step, since it has no `Call` method. But Go lets a +function type have a method (`type FilterFunc func(...) error`, as +`http.HandlerFunc`). **The runtime gives no such adapter.** A user that writes +one gets the label `FilterFunc`; the linter flags it (`CUP-A001`). + +### D7. A named pipeline + +**Recommendation.** As Rust: an interface with one method that adds the steps. +The label is the name of the type. + +```go +type PricingPipeline struct{} + +func (PricingPipeline) AddSteps(pipeline *codeupipe.Pipeline) { + pipeline.AddFilter(DiscountFilter{Rate: 0.1}) + // MarkBulkFilter runs only when the predicate is true. + pipeline.AddValve(AtLeastPredicate{Key: "qty", Min: 10}, MarkBulkFilter{}) + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.AddTap(AuditTap{}) +} + +pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) +``` + +`AddSteps` is also what the compiler reads (phase 3). A plain +`codeupipe.NewPipeline()` has the label `Pipeline`. + +### D8. Run, State, and goroutines + +**Recommendation.** `Run(ctx, input) (*Payload, error)` is sync, which is all +that Go needs: a step that waits blocks its goroutine, and a caller that wants +runs at the same time starts goroutines. Go has no async functions, so there is +no `RunAsync` and no `AsyncStepInSyncRun`. Principle 3 holds by itself. + +Goroutines really do run at the same time, so "a State belongs to one run" is +not only for overlap inside one thread, as it is in JavaScript. + +- `RunWithState(ctx, input, state)` is the run with the caller's State (Go + has no optional argument; the name is Rust's). +- `pipeline.State()` is the State of the run that started last, once it has + ended (Rust's rule). Each run takes a number from an atomic counter when it + starts, and its State is kept at the end only if no later run has started. +- **A Pipeline can be run from many goroutines at the same time.** Adding a + step while it runs is not supported; the README says so. +- The tests run with `go test -race`, in CI too. + +Rust keeps the storage of a State between runs for speed. Go does that only if +the benchmark shows a need. + +### D9. A type of your own + +**The question.** The contract lets a pipeline carry a type of your own in +place of the Payload. Go has generics (1.18), but **no default type +parameter**, so Rust's `Pipeline

` has no Go form. + +**Recommendation.** Each primitive has a generic form, `...Of[P]`, and an +alias for the Payload with the short name. The alias is the same type, not a +second one, so principle 1 holds. + +```go +type FilterOf[P any] interface { + Call(ctx context.Context, payload P) error +} + +type Filter = FilterOf[*Payload] +type Pipeline = PipelineOf[*Payload] +``` + +Code for the Payload never writes `Of`. A type of your own: + +```go +pipeline := codeupipe.NewPipelineOf[*Order]() +pipeline.AddFilter(DiscountOrderFilter{}) // a FilterOf[*Order] +``` + +`P` must have `Copy() P`, for the copy that `run` works on (`PipelineData[P]`). +Lineage is kept only by a type that also has `Stamp(label string)`. A step for +one type in a pipeline of another does not compile. + +The alias of an instantiated generic type is checked on Go 1.25 with a +`go 1.22` module. + +### D10. The Go version + +**Recommendation.** `go 1.22` in `go.mod`. Generics need 1.18. From 1.22, each +iteration of a loop has its own variable, which removes a class of goroutine +bugs in the code of the user. CI tests 1.22 and the newest Go. This machine has +1.25. + +### D11. Names + +**Recommendation.** The module is +`github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go`, and the +package is `codeupipe`, so the code reads `codeupipe.Payload`. A user who wants +it short imports it as `cup`. The directory and the package have different +names, which Go allows (`gopkg.in/yaml.v3` is the package `yaml`). + +The names are Go's: `AddFilter`, `AddValve`, `AddTap`, `AddPipeline`, +`UseHook`, `Run`, `RunWithState`, `State`, `Labels`, `Observe`, `Insert`, +`Remove`, `Merge`, `Copy`, `ToDict`, `WithTrace`, `TraceID`, `Lineage`, +`OnError`. A constructor is `New...`: `NewPayload(map)`, `NewPipeline()`, +`NewNamedPipeline(p)`, `NewState()`. + +### D12. Copy + +**Recommendation.** As the contract and TypeScript: `Copy()` is shallow, and +`NewPayload(map)` copies the map that it is given, not the values in it. Go's +maps and slices are references, so this is exactly "the two payloads have +separate keys and share their values". Rust had to make a deep copy; Go does +not. + +## Phase 1: the core package + +**Done** (2026-10-29): `packages/codeupipe-go`, 725 lines of source with +their doc comments, and 60 tests in 1,001 lines (18 Payload, 23 Pipeline, 9 +labels, 6 a type of your own, 1 of what does not compile, 3 the example). `go test -race`, `go vet`, +`gofmt` and `staticcheck` are clean on Go 1.22.12 and 1.25.0. The tests are +ported from `packages/codeupipe-ts/tests`, except the async ones, which have +no Go form, and add what only Go has: `ctx` reaches each step, a cancelled +`ctx` does not stop a run, 50 runs in goroutines at the same time, and a nil +step refused when it is added. To check that the tests catch a broken +runtime, two rules were broken on purpose: keeping the State of a run that +did not start last failed 1 test, and not recording a Tap failure failed 2. + +What the contract refuses when a step is added is a compile error, and +`refused_test.go` checks it as Rust's `compile_fail` tests do: it builds six +small programs (a function as a step, a Filter as a Tap, `AddValve` in the +wrong order, a step for another type, a run of another type, a type with no +`Copy`) and checks that each build fails with the error that names the cause. +A seventh program with the right steps must compile, so that a failure comes +from the shape. + +The eight primitives, to the contract, with the decisions above. + +1. `payload.go`, tests first (RED, GREEN, REFACTOR), from the Payload table + of the contract and `packages/codeupipe-ts/tests`. +2. The interfaces: `FilterOf`, `PredicateOf`, `TapOf`, `HookOf`. + `step_label.go`. +3. `State`, the valve, `PipelineOf` (`AddFilter`, `AddValve`, `AddTap`, + `AddPipeline`, `UseHook`, `Run`, `RunWithState`, `State`, `Labels`, + `Observe`), and `NamedPipeline`. +4. The pricing example, with its test, giving + `{"price": 90, "qty": 12, "bulk": true}`. +5. The README, in the form of the TypeScript and Rust READMEs. + +Done when: `go test -race ./...`, `go vet ./...` and `gofmt -l .` (no output) +pass on Go 1.22 and on the newest Go, and `go run ./examples/pricing` prints +the result and the State. + +## Phase 2: conformance + +**Done** (2026-10-29): `go:run` passes all 30 cases, with nothing +unsupported, and every other target still passes in the same run of +`run.py`. To check that the runner catches a broken runtime, three rules were +broken on purpose: not recording a Tap failure failed 4 cases, keeping the +State of a run that did not start last failed 1, and not sending `OnError` +for the pipeline failed 2. The runner is 523 lines (`main.go`, `bench.go`) +and 19 files in `cup_ops/` (402 lines), with `encoding/json` and no JSON +reader of its own. The judge has a `GoProbe` and `run.py` leaves `go:run` +out on a machine with no `go`; `bench.py` keeps its baseline under `go`; CI +has a `go` job (gofmt, vet, staticcheck, `go test -race`, the example, +`go:run`, and Go 1.22); and API.md has a Go column and notes. + +The first numbers showed a 10-step run at 0.69 µs. A Go benchmark of 10 +no-op steps showed 9 allocations a run, 4 of them the list of executed +labels in State, which grew at each step. It is now made once with room for +every step: 10 no-op steps went from 356 to 217 ns. What is left is mostly +the number that each `add` writes: an `any` holds a `float64` in an +allocation of its own (D1). + +µs per run, Apple Silicon, one run of `python conformance/bench.py`: + +| scenario | python:run | python:static-inline | rust:run | go:run | +|---|---|---|---|---| +| `micro-pass` (10 filters) | 4.79 | 0.627 | 0.276 | 0.546 | +| `valve-gated` | 5.14 | 0.860 | 0.326 | 0.612 | +| `taps` | 5.03 | 0.626 | 0.293 | 0.567 | +| `hook` | 5.41 | — | 0.286 | 0.626 | +| `heavy-allocation` (50 inserts) | 15.03 | 1.51 | 4.40 | 2.15 | + +The dynamic Go run is 9 times faster than `python:run`, about as fast as +compiled Python, and half as fast as Rust, except where a run makes 50 new +keys: there Go's map is twice as fast as the Rust runtime's. + +What follows is the plan as it was written. + +A runner in `conformance/runners/go/`, with the protocol of the others: +`runner < cases.json > results.json`. It never judges. + +- **Steps.** `cup_ops/` has one file for each step type, with the names of the + TypeScript and Rust runners: `SetFilter`, `CopyFilter`, `AddFilter`, + `AppendFilter`, `RemoveFilter`, `MergeFilter`, `HasFilter`, `MutateFilter`, + `FailFilter`, `RunAgainFilter`, `RecordTap`, `FailTap`, `NoopTap`, + `EqualsPredicate`, `TruthyPredicate`, `RecordHook`, `NoopHook`, + `NestedPipeline`. +- **JSON.** `encoding/json`. No reader of its own. +- **Offline.** Its own `go.mod`, with a `replace` to the package. `run.py` + runs it with `GOFLAGS=-mod=mod GOPROXY=off`, so it cannot download. With no + `go` on the machine, `run.py` leaves the target out, as it does for `cargo`. +- **`equals` and `truthy`** (open questions 3 and 4 of the conformance + README): as Rust. `equals` compares by content, and a number is never equal + to a bool. `truthy` follows Python: an empty list or map is false. +- **Benchmark.** `go:run` in `bench.py`, with a reference of map writes of its + own, as Rust has. +- **CI.** A `go` job: `gofmt`, `go vet`, `go test -race`, the example, + `python conformance/run.py go:run`, and the tests again on Go 1.22. +- **Documents.** A Go column in the "Runtimes" table of API.md, a row in the + README's table of packages, and `go:run` in MATRIX.md. + +Done when: `go:run` passes all 30 cases, with nothing unsupported. To check +that the runner catches a broken runtime, the runtime is broken on purpose +(a Tap failure not recorded) and cases must fail. + +Phases 1 and 2 are one release: `codeupipe` 0.1.0 for Go, in the conformance +matrix. + +## Phase 3: compile, and phase 4: the linter + +Later versions, each with a plan of its own when it starts. Both use only the +standard library (`go/parser`, `go/ast`, `go/types`, `go/format`). The compiler +starts only after phase 2 measures `go:run`, as the Rust compiler waited for +`rust:run`: it is worth writing only if the number leaves room for a gain. + +## Order and size + +| Phase | What | Rough size | +|---|---|---| +| 0 | Settle D1 to D12. **Done** (2026-10-29) | One review of this page | +| 1 | Core package, example, README | About 500 lines of source and 600 of tests (TypeScript: 450 lines of source) | +| 2 | Runner, `run.py`, bench, CI, documents | About 300 lines: no JSON reader to write | +| 3 | Compiler | Later | +| 4 | Linter | Later | diff --git a/packages/codeupipe-go/LICENSE b/packages/codeupipe-go/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-go/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-go/NOTICE b/packages/codeupipe-go/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-go/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-go/README.md b/packages/codeupipe-go/README.md new file mode 100644 index 0000000..6152e4f --- /dev/null +++ b/packages/codeupipe-go/README.md @@ -0,0 +1,209 @@ +# codeupipe for Go + +codeupipe is a library for writing a program as a pipeline of small steps. +This is the Go runtime. It has no dependencies. + +It implements the same contract as the Python, TypeScript and Rust packages: +[conformance/API.md](../../conformance/API.md), and passes all its +conformance cases (`python conformance/run.py go:run`). There is no linter +and no compiler for Go yet ([docs/go-plan.md](../../docs/go-plan.md)). + +The version is below 1.0: the package is not stable yet, and a new version +can change in a way that is not compatible. + +No version is tagged yet. Use it from the main branch: + +```bash +go get github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go@main +``` + +It needs Go 1.22 or later. + +## The eight primitives + +| Primitive | What it does | Who writes it | +|---|---|---| +| **Payload** | Carries the data | The library | +| **Filter** | Changes the data | You | +| **Predicate** | Asks a yes-or-no question about the data | You | +| **Valve** | Joins a Predicate to a Filter: the Filter runs when the answer is yes | The library. You add one with `AddValve(predicate, filter)` | +| **Tap** | Reads the data as it goes by | You | +| **Hook** | Is called before and after each step | You | +| **State** | Records which steps ran | The library | +| **Pipeline** | Runs the steps in order | You | + +## Example + +[examples/pricing/](examples/pricing/) has one type in each file. The +pipeline file adds them. + +```go +// discount_filter.go +type DiscountFilter struct { + Rate float64 +} + +func (f DiscountFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + price, _ := payload.Get("price").(float64) + payload.Insert("price", math.Round(price*(1-f.Rate)*100)/100) + return nil +} +``` + +```go +// at_least_predicate.go +type AtLeastPredicate struct { + Key string + Minimum float64 +} + +func (q AtLeastPredicate) Test(ctx context.Context, payload *codeupipe.Payload) bool { + value, _ := payload.Get(q.Key).(float64) + return value >= q.Minimum +} +``` + +```go +// pricing_pipeline.go +type PricingPipeline struct{} + +func (PricingPipeline) AddSteps(pipeline *codeupipe.Pipeline) { + pipeline.AddFilter(DiscountFilter{Rate: 0.1}) + // MarkBulkFilter runs only when the predicate is true. + pipeline.AddValve(AtLeastPredicate{Key: "qty", Minimum: 10}, MarkBulkFilter{}) + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.AddTap(AuditTap{}) +} +``` + +```go +pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) +result, err := pipeline.Run(ctx, codeupipe.NewPayload(map[string]any{"price": 100, "qty": 12})) +result.ToDict() +// map[bulk:true price:90 qty:12] +``` + +```bash +go run ./examples/pricing +``` + +## A payload of your own + +`Payload` is the default, and it stays the default: a map of JSON-shaped +values that any developer can read, in any runtime of codeupipe. It shows +that anything can go there. + +You can opt in to a type of your own, such as the struct that you would +write anyway. Give it a `Copy` method, and the steps of that pipeline take +the struct: + +```go +type Order struct { + Price float64 + Qty float64 + Bulk bool +} + +func (o *Order) Copy() *Order { + copied := *o + return &copied +} + +type MarkBulkFilter struct{} + +func (MarkBulkFilter) Call(ctx context.Context, order *Order) error { + order.Bulk = true + return nil +} + +pipeline := codeupipe.NewPipelineOf[*Order]() +pipeline.AddValve(BulkPredicate{}, MarkBulkFilter{}) +``` + +Each primitive has a generic form, `FilterOf[P]`, `PredicateOf[P]`, +`TapOf[P]`, `HookOf[P]`, `NamedPipelineOf[P]` and `PipelineOf[P]`, and a +short name for the Payload: `Filter` is `FilterOf[*Payload]`, the same type. +Go has no default type parameter, so code for the Payload never writes `Of`. +The rules stay the same: `Run` works on a copy of the input, only a Filter +writes, a Tap failure is recorded and the run goes on, and the labels are the +same. A step for another type does not compile. Lineage is kept only if the +type also has `Stamp(label string)`. + +## Names + +The names are the names of the contract, in the case of Go. + +| Contract | Go | +|---|---| +| `add_filter`, `add_valve`, `add_tap`, `add_pipeline`, `use_hook` | `AddFilter`, `AddValve`, `AddTap`, `AddPipeline`, `UseHook` | +| `run(p)`, `run(p, state)`, `state`, `labels` | `Run(ctx, p)`, `RunWithState(ctx, p, state)`, `State()`, `Labels()` | +| `observe(timing, lineage)` | `Observe(timing, lineage)`, two `bool`s | +| `get`, `has`, `insert`, `remove`, `merge`, `copy`, `to_dict`, `with_trace` | `Get`, `Has`, `Insert`, `Remove`, `Merge`, `Copy`, `ToDict`, `WithTrace` | +| `trace_id`, `lineage` | `TraceID()`, `Lineage()` | +| Hook: `before`, `after`, `on_error` | `Before`, `After`, `OnError` | +| construct | `NewPayload(map)`, `NewPipeline()`, `NewState()` | +| A named pipeline: `class PricingPipeline(Pipeline)` | A type with `AddSteps(pipeline *codeupipe.Pipeline)`, then `codeupipe.NewNamedPipeline(PricingPipeline{})` | + +`Filter`, `Predicate`, `Tap` and `Hook` are interfaces. Embed +`codeupipe.HookBase` in a Hook, and write only the methods that you need. + +## Differences from Python, TypeScript and Rust + +- **`ctx` comes first.** Each step method and `Run` take a + `context.Context`, as Go code that can wait does. The Pipeline gives it to + each step and does nothing else with it: a cancelled `ctx` does not stop a + run between steps. A step that calls the network or a database gives it + the caller's `ctx`. +- **A value is an `any`**, in the shapes that `encoding/json` gives: `nil`, + `bool`, `float64`, `string`, `[]any`, `map[string]any`. There is one number + type: `Insert` and `NewPayload` keep an `int` (or any other Go number) as a + `float64`, so `payload.Insert("qty", 12)` is read back with + `.(float64)`. A list or a map is kept as it is given. +- **`Get` gives `nil` when a key is absent.** `Has` tells a key with a nil + value from an absent key. A default is a type assertion: + `price, ok := payload.Get("price").(float64)`. +- **The keys have no order**, as a Go map has none. `fmt` prints a map with + its keys sorted. +- **A Filter changes the Payload in place** and returns `nil`, as the Payload + is changed in place in TypeScript and Rust. A Hook's `OnError` sees the + Payload as the failing step left it. A Tap, a Predicate and a Hook also get + `*Payload`: Go has no pointer that only reads, so they must not write. As in + Python and TypeScript, that is a rule for the linter, not for the compiler. +- **What the contract refuses when a step is added is a compile error.** A + function is not a Filter, and `AddValve(filter, predicate)` does not + compile. `refused_test.go` builds each shape and checks the error. A nil + step compiles; it is refused with a panic when it is added. +- **`Run` is sync, and there is no `RunAsync`.** Go needs none: a step that + waits blocks its goroutine. **A Pipeline can be run from many goroutines at + the same time**, and each run writes into a State of its own. Add the steps + before the first run. +- **`State()` is the State of the run that started last, once it has + ended**, as in Rust. The run with the caller's State is + `RunWithState(ctx, input, state)`. +- **A failure is an `error`.** `Run` returns the error of the step itself, so + `errors.Is` and `errors.As` work, and `nil` for the data. A panic is not a + failure: it is not caught, and a panicking Tap stops the run. +- **`Copy()` is shallow**, as the contract says: Go's maps and slices are + references, so two Payloads share their values. +- **The label** is the name of the step's type, from `reflect`, without + its package or type arguments. For a pointer it is the type that it points + to. A struct type with no name gives `anonymous`. +- **The trace id** is `""` until it is set. + +## Development + +In this directory: + +```bash +go test -race ./... +go vet ./... +gofmt -l . +go run ./examples/pricing +``` + +One primitive is in each file, and the file name is the type name in +snake_case. Each test is beside the file that it tests. + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-go/doc.go b/packages/codeupipe-go/doc.go new file mode 100644 index 0000000..0a48694 --- /dev/null +++ b/packages/codeupipe-go/doc.go @@ -0,0 +1,15 @@ +// Package codeupipe writes a program as a pipeline of small steps. +// +// A Payload carries the data. A Filter changes it, a Predicate asks a +// yes-or-no question about it, a Valve joins a Predicate to a Filter, a Tap +// reads it as it goes by, a Hook sees the lifecycle of a run, and State +// records what happened. A Pipeline runs the steps in order. +// +// The contract of these eight primitives is conformance/API.md in the +// repository. The Python, TypeScript and Rust runtimes have the same one, +// and the same conformance cases judge each of them. +// +// Each step method and Run take a context.Context first. The Pipeline gives +// it to each step and does nothing else with it: a cancelled context does +// not stop a run between steps. +package codeupipe diff --git a/packages/codeupipe-go/examples/pricing/at_least_predicate.go b/packages/codeupipe-go/examples/pricing/at_least_predicate.go new file mode 100644 index 0000000..03be33d --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/at_least_predicate.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// AtLeastPredicate is a Predicate: it asks a yes-or-no question and changes +// nothing. Its settings are fields. +type AtLeastPredicate struct { + Key string + Minimum float64 +} + +func (q AtLeastPredicate) Test(ctx context.Context, payload *codeupipe.Payload) bool { + value, _ := payload.Get(q.Key).(float64) + return value >= q.Minimum +} diff --git a/packages/codeupipe-go/examples/pricing/audit_tap.go b/packages/codeupipe-go/examples/pricing/audit_tap.go new file mode 100644 index 0000000..0d9e421 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/audit_tap.go @@ -0,0 +1,17 @@ +package main + +import ( + "context" + "fmt" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// AuditTap is a Tap: it reads the Payload and changes nothing. If it fails, +// State records the error and the run goes on. +type AuditTap struct{} + +func (AuditTap) Observe(ctx context.Context, payload *codeupipe.Payload) error { + fmt.Println("audit:", payload.ToDict()) + return nil +} diff --git a/packages/codeupipe-go/examples/pricing/discount_filter.go b/packages/codeupipe-go/examples/pricing/discount_filter.go new file mode 100644 index 0000000..a0a8952 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/discount_filter.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + "math" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// DiscountFilter is a Filter. Its setting is a field. +type DiscountFilter struct { + Rate float64 +} + +func (f DiscountFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + price, _ := payload.Get("price").(float64) + payload.Insert("price", math.Round(price*(1-f.Rate)*100)/100) + return nil +} diff --git a/packages/codeupipe-go/examples/pricing/main.go b/packages/codeupipe-go/examples/pricing/main.go new file mode 100644 index 0000000..d93b784 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/main.go @@ -0,0 +1,27 @@ +// go run ./examples/pricing: run the pipeline with a Hook and show its State. +package main + +import ( + "context" + "fmt" + "log" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +func main() { + pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) + pipeline.UseHook(PrintStepsHook{}) + + fmt.Println("steps:") + result, err := pipeline.Run(context.Background(), codeupipe.NewPayload(map[string]any{"price": 100, "qty": 12})) + if err != nil { + log.Fatal(err) + } + + state := pipeline.State() + fmt.Println("result: ", result.ToDict()) + fmt.Println("executed:", state.Executed()) + fmt.Println("skipped: ", state.Skipped()) + fmt.Println("errors: ", state.Errors()) +} diff --git a/packages/codeupipe-go/examples/pricing/mark_bulk_filter.go b/packages/codeupipe-go/examples/pricing/mark_bulk_filter.go new file mode 100644 index 0000000..e8133b5 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/mark_bulk_filter.go @@ -0,0 +1,15 @@ +package main + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// MarkBulkFilter is a Filter. +type MarkBulkFilter struct{} + +func (MarkBulkFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { + payload.Insert("bulk", true) + return nil +} diff --git a/packages/codeupipe-go/examples/pricing/pricing_pipeline.go b/packages/codeupipe-go/examples/pricing/pricing_pipeline.go new file mode 100644 index 0000000..e774359 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/pricing_pipeline.go @@ -0,0 +1,14 @@ +package main + +import "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" + +// PricingPipeline is a named pipeline: codeupipe.NewNamedPipeline(PricingPipeline{}). +type PricingPipeline struct{} + +func (PricingPipeline) AddSteps(pipeline *codeupipe.Pipeline) { + pipeline.AddFilter(DiscountFilter{Rate: 0.1}) + // MarkBulkFilter runs only when the predicate is true. + pipeline.AddValve(AtLeastPredicate{Key: "qty", Minimum: 10}, MarkBulkFilter{}) + // A Tap only reads. If it fails, State records the error and the run goes on. + pipeline.AddTap(AuditTap{}) +} diff --git a/packages/codeupipe-go/examples/pricing/pricing_pipeline_test.go b/packages/codeupipe-go/examples/pricing/pricing_pipeline_test.go new file mode 100644 index 0000000..8881d67 --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/pricing_pipeline_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "reflect" + "testing" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +func same(t *testing.T, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v, want %#v", got, want) + } +} + +func TestALargeOrderGetsTheDiscountAndTheBulkMark(t *testing.T) { + pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) + out, err := pipeline.Run(context.Background(), codeupipe.NewPayload(map[string]any{"price": 100, "qty": 12})) + same(t, err, nil) + same(t, out.ToDict(), map[string]any{"price": 90.0, "qty": 12.0, "bulk": true}) + same(t, pipeline.State().Executed(), []string{"DiscountFilter", "MarkBulkFilter", "AuditTap"}) + same(t, pipeline.State().Errors(), []codeupipe.StepFailure{}) +} + +func TestASmallOrderGetsTheDiscountOnly(t *testing.T) { + pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) + out, err := pipeline.Run(context.Background(), codeupipe.NewPayload(map[string]any{"price": 100, "qty": 2})) + same(t, err, nil) + same(t, out.ToDict(), map[string]any{"price": 90.0, "qty": 2.0}) + same(t, pipeline.State().Skipped(), []string{"MarkBulkFilter"}) +} + +func TestTheTapReadsThePayload(t *testing.T) { + err := AuditTap{}.Observe(context.Background(), codeupipe.NewPayload(map[string]any{"price": 90})) + same(t, err, nil) +} diff --git a/packages/codeupipe-go/examples/pricing/print_steps_hook.go b/packages/codeupipe-go/examples/pricing/print_steps_hook.go new file mode 100644 index 0000000..fb4933d --- /dev/null +++ b/packages/codeupipe-go/examples/pricing/print_steps_hook.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "fmt" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// PrintStepsHook is a Hook: it sees the lifecycle. step is the label of the +// step, which is the name of its type. It is "" for the pipeline itself. +// HookBase gives the methods that it does not write. +type PrintStepsHook struct { + codeupipe.HookBase +} + +func (PrintStepsHook) Before(ctx context.Context, step string, payload *codeupipe.Payload) { + if step != "" { + fmt.Println(" ", step) + } +} diff --git a/packages/codeupipe-go/filter.go b/packages/codeupipe-go/filter.go new file mode 100644 index 0000000..f606247 --- /dev/null +++ b/packages/codeupipe-go/filter.go @@ -0,0 +1,19 @@ +package codeupipe + +import "context" + +// FilterOf changes the data of a pipeline that carries P. Call changes the +// data in place. A failure is an error, and it stops the run. +type FilterOf[P any] interface { + Call(ctx context.Context, payload P) error +} + +// Filter changes the Payload. It is the only step that writes. +// +// type MarkBulkFilter struct{} +// +// func (MarkBulkFilter) Call(ctx context.Context, payload *codeupipe.Payload) error { +// payload.Insert("bulk", true) +// return nil +// } +type Filter = FilterOf[*Payload] diff --git a/packages/codeupipe-go/go.mod b/packages/codeupipe-go/go.mod new file mode 100644 index 0000000..eedba80 --- /dev/null +++ b/packages/codeupipe-go/go.mod @@ -0,0 +1,3 @@ +module github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go + +go 1.22 diff --git a/packages/codeupipe-go/hook.go b/packages/codeupipe-go/hook.go new file mode 100644 index 0000000..e7a5900 --- /dev/null +++ b/packages/codeupipe-go/hook.go @@ -0,0 +1,41 @@ +package codeupipe + +import "context" + +// HookOf sees the lifecycle of a run of a pipeline that carries P. It reads +// the data and does not write to it. +// +// step is the label of the step, or "" for the pipeline itself. On success: +// Before(""), then Before and After for each Filter or Valve, then +// After(""). On a Filter failure: OnError for the step, then OnError(""), +// and no After. Taps get no events. +type HookOf[P any] interface { + Before(ctx context.Context, step string, payload P) + After(ctx context.Context, step string, payload P) + OnError(ctx context.Context, step string, err error, payload P) +} + +// Hook sees the lifecycle of a run of a pipeline that carries a Payload. +type Hook = HookOf[*Payload] + +// HookBaseOf has the three methods of a Hook, and they do nothing. Embed it, +// and write only the methods that you need. +type HookBaseOf[P any] struct{} + +// Before does nothing. +func (HookBaseOf[P]) Before(context.Context, string, P) {} + +// After does nothing. +func (HookBaseOf[P]) After(context.Context, string, P) {} + +// OnError does nothing. +func (HookBaseOf[P]) OnError(context.Context, string, error, P) {} + +// HookBase is HookBaseOf for a pipeline that carries a Payload. +// +// type PrintStepsHook struct{ codeupipe.HookBase } +// +// func (PrintStepsHook) Before(ctx context.Context, step string, payload *codeupipe.Payload) { +// fmt.Println(step) +// } +type HookBase = HookBaseOf[*Payload] diff --git a/packages/codeupipe-go/named_pipeline.go b/packages/codeupipe-go/named_pipeline.go new file mode 100644 index 0000000..985d5f3 --- /dev/null +++ b/packages/codeupipe-go/named_pipeline.go @@ -0,0 +1,35 @@ +package codeupipe + +// NamedPipelineOf is a pipeline type of your own, for a pipeline that +// carries P. AddSteps adds its steps, and its label is the name of the type. +type NamedPipelineOf[P PipelineData[P]] interface { + AddSteps(pipeline *PipelineOf[P]) +} + +// NamedPipeline is a pipeline type of your own. Nested in another pipeline, +// its label says what it is. +// +// type PricingPipeline struct{} +// +// func (PricingPipeline) AddSteps(pipeline *codeupipe.Pipeline) { +// pipeline.AddFilter(DiscountFilter{Rate: 0.1}) +// pipeline.AddValve(AtLeastPredicate{Key: "qty", Minimum: 10}, MarkBulkFilter{}) +// pipeline.AddTap(AuditTap{}) +// } +// +// pipeline := codeupipe.NewNamedPipeline(PricingPipeline{}) +type NamedPipeline = NamedPipelineOf[*Payload] + +// NewNamedPipeline gives the pipeline that named adds the steps of. Its +// label is the name of the type of named. +func NewNamedPipeline(named NamedPipeline) *Pipeline { + return NewNamedPipelineOf[*Payload](named) +} + +// NewNamedPipelineOf is NewNamedPipeline for a pipeline that carries P. +func NewNamedPipelineOf[P PipelineData[P]](named NamedPipelineOf[P]) *PipelineOf[P] { + pipeline := NewPipelineOf[P]() + pipeline.label = labelOf(named) + named.AddSteps(pipeline) + return pipeline +} diff --git a/packages/codeupipe-go/payload.go b/packages/codeupipe-go/payload.go new file mode 100644 index 0000000..8f4d2ce --- /dev/null +++ b/packages/codeupipe-go/payload.go @@ -0,0 +1,148 @@ +package codeupipe + +import "maps" + +// Payload is the data that moves through a pipeline. +// +// One keyed container. Keys are strings, and values are JSON-shaped, in the +// shapes that encoding/json gives: nil, bool, float64, string, []any and +// map[string]any. Every write changes the Payload in place and returns it, +// so writes chain. Copy gives a separate Payload when you want to keep the +// one you have. +// +// The zero Payload is empty and ready to use. +type Payload struct { + data map[string]any + traceID string + lineage []string +} + +// NewPayload gives a Payload with the keys of data. The map is copied, so +// the Payload does not change it. It can be nil. +func NewPayload(data map[string]any) *Payload { + payload := &Payload{data: make(map[string]any, len(data))} + for key, value := range data { + payload.data[key] = number(value) + } + return payload +} + +// Get gives the value of key, or nil when key is absent. A default is a type +// assertion: price, ok := payload.Get("price").(float64). +func (p *Payload) Get(key string) any { + return p.data[key] +} + +// Has tells whether key is present, also when its value is nil. +func (p *Payload) Has(key string) bool { + _, ok := p.data[key] + return ok +} + +// Insert sets key to value in this Payload, and returns it. A number of +// another Go type (int, int64, float32 and the others) is kept as a float64. +func (p *Payload) Insert(key string, value any) *Payload { + if p.data == nil { + p.data = make(map[string]any) + } + p.data[key] = number(value) + return p +} + +// Remove removes key from this Payload (nothing happens if it is absent), +// and returns it. +func (p *Payload) Remove(key string) *Payload { + delete(p.data, key) + return p +} + +// Merge writes every key of other into this Payload (other wins), and +// returns it. It keeps its trace id, or takes the trace id of other when it +// has none, and adds the lineage of other to its own. +func (p *Payload) Merge(other *Payload) *Payload { + if p.data == nil { + p.data = make(map[string]any, len(other.data)) + } + for key, value := range other.data { + p.data[key] = value + } + if p.traceID == "" { + p.traceID = other.traceID + } + p.lineage = append(p.lineage, other.lineage...) + return p +} + +// Copy gives a separate Payload with the same contents, trace id and +// lineage. The copy is shallow: the two Payloads have separate keys and +// share their values. Copy a nested list or map before you change it. +func (p *Payload) Copy() *Payload { + data := maps.Clone(p.data) + if data == nil { + data = make(map[string]any) + } + return &Payload{data: data, traceID: p.traceID, lineage: append([]string(nil), p.lineage...)} +} + +// ToDict gives the contents as a new map. +func (p *Payload) ToDict() map[string]any { + if p.data == nil { + return map[string]any{} + } + return maps.Clone(p.data) +} + +// WithTrace sets the trace id on this Payload, and returns it. +func (p *Payload) WithTrace(traceID string) *Payload { + p.traceID = traceID + return p +} + +// TraceID gives the trace id, for following one Payload across systems. It +// is "" until it is set. +func (p *Payload) TraceID() string { + return p.traceID +} + +// Lineage gives the labels of the steps that this Payload has passed +// through, when lineage is on (Pipeline.Observe). +func (p *Payload) Lineage() []string { + return append([]string{}, p.lineage...) +} + +// Stamp records a step in the lineage. The Pipeline calls it. +func (p *Payload) Stamp(label string) { + p.lineage = append(p.lineage, label) +} + +// number gives a number of any Go type as a float64, the one number type of +// a Payload, and any other value as it is. +func number(value any) any { + switch n := value.(type) { + case int: + return float64(n) + case int8: + return float64(n) + case int16: + return float64(n) + case int32: + return float64(n) + case int64: + return float64(n) + case uint: + return float64(n) + case uint8: + return float64(n) + case uint16: + return float64(n) + case uint32: + return float64(n) + case uint64: + return float64(n) + case uintptr: + return float64(n) + case float32: + return float64(n) + } + return value +} diff --git a/packages/codeupipe-go/payload_test.go b/packages/codeupipe-go/payload_test.go new file mode 100644 index 0000000..025d305 --- /dev/null +++ b/packages/codeupipe-go/payload_test.go @@ -0,0 +1,141 @@ +package codeupipe_test + +import ( + "reflect" + "testing" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +func same(t *testing.T, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v, want %#v", got, want) + } +} + +func TestTheMapThatTheCallerGivesIsCopied(t *testing.T) { + data := map[string]any{"a": 1.0} + payload := codeupipe.NewPayload(data) + payload.Insert("b", 2.0) + same(t, data, map[string]any{"a": 1.0}) + same(t, payload.ToDict(), map[string]any{"a": 1.0, "b": 2.0}) +} + +func TestAnEmptyPayloadHasNoKeys(t *testing.T) { + same(t, codeupipe.NewPayload(nil).ToDict(), map[string]any{}) +} + +func TestGetGivesTheValueOrNil(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0, "n": nil}) + same(t, payload.Get("a"), 1.0) + same(t, payload.Get("missing"), nil) + same(t, payload.Get("n"), nil) +} + +func TestADefaultIsATypeAssertion(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"price": 90.0, "name": "Ada"}) + price, ok := payload.Get("price").(float64) + same(t, []any{price, ok}, []any{90.0, true}) + qty, ok := payload.Get("qty").(float64) // absent + same(t, []any{qty, ok}, []any{0.0, false}) + count, ok := payload.Get("name").(float64) // not a number + same(t, []any{count, ok}, []any{0.0, false}) +} + +func TestHasIsTrueForAKeyWithANilValue(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"n": nil}) + same(t, payload.Has("n"), true) + same(t, payload.Has("missing"), false) +} + +func TestInsertRemoveMergeAndWithTraceChangeThePayloadAndReturnIt(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0}) + same(t, payload.Insert("b", 2.0) == payload, true) + same(t, payload.Remove("a") == payload, true) + same(t, payload.Merge(codeupipe.NewPayload(map[string]any{"c": 3.0})) == payload, true) + same(t, payload.WithTrace("t-1") == payload, true) + same(t, payload.ToDict(), map[string]any{"b": 2.0, "c": 3.0}) + same(t, payload.TraceID(), "t-1") +} + +func TestWritesChain(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0}) + payload.Insert("b", 2.0).Remove("a").Insert("c", 3.0) + same(t, payload.ToDict(), map[string]any{"b": 2.0, "c": 3.0}) +} + +func TestRemoveOfAKeyThatIsNotThereDoesNothing(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0}).Remove("zz") + same(t, payload.ToDict(), map[string]any{"a": 1.0}) +} + +func TestInAMergeTheOtherPayloadWins(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0, "b": 1.0}) + payload.Merge(codeupipe.NewPayload(map[string]any{"b": 2.0})) + same(t, payload.ToDict(), map[string]any{"a": 1.0, "b": 2.0}) +} + +func TestAMergeKeepsTheTraceIDItHasAndTakesOneWhenItHasNone(t *testing.T) { + other := codeupipe.NewPayload(nil).WithTrace("other") + same(t, codeupipe.NewPayload(nil).WithTrace("mine").Merge(other).TraceID(), "mine") + same(t, codeupipe.NewPayload(nil).Merge(other).TraceID(), "other") +} + +func TestCopyGivesASeparatePayloadWithTheSameContents(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0}).WithTrace("t-1") + copied := payload.Copy() + copied.Insert("a", 2.0) + same(t, payload.Get("a"), 1.0) + same(t, copied.TraceID(), "t-1") +} + +func TestCopyIsShallowTheTwoPayloadsShareTheirValues(t *testing.T) { + list := []any{1.0} + payload := codeupipe.NewPayload(map[string]any{"list": list}) + shared := payload.Copy().Get("list").([]any) + same(t, &shared[0] == &list[0], true) +} + +func TestToDictGivesANewMapEachTime(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"a": 1.0}) + payload.ToDict()["a"] = 2.0 + same(t, payload.Get("a"), 1.0) +} + +func TestTheTraceIDIsEmptyUntilItIsSetAndTheLineageStartsEmpty(t *testing.T) { + payload := codeupipe.NewPayload(nil) + same(t, payload.TraceID(), "") + same(t, payload.Lineage(), []string{}) +} + +func TestTheLineageIsACopy(t *testing.T) { + payload := codeupipe.NewPayload(nil) + payload.Stamp("AFilter") + payload.Lineage()[0] = "changed" + same(t, payload.Lineage(), []string{"AFilter"}) +} + +// D1: one number type. Go's other number types become float64, so a value +// that is inserted as 12 is read back with .(float64). +func TestANumberIsAFloat64(t *testing.T) { + payload := codeupipe.NewPayload(map[string]any{"given": 7}) + payload.Insert("int", 12).Insert("int64", int64(-3)).Insert("uint8", uint8(200)).Insert("float32", float32(0.5)) + same(t, payload.ToDict(), map[string]any{ + "given": 7.0, "int": 12.0, "int64": -3.0, "uint8": 200.0, "float32": 0.5, + }) +} + +func TestAListOrAMapIsKeptAsItIsGiven(t *testing.T) { + payload := codeupipe.NewPayload(nil).Insert("list", []any{1}).Insert("ints", []int{1}) + same(t, payload.Get("list"), []any{1}) + same(t, payload.Get("ints"), []int{1}) +} + +func TestTheZeroPayloadCanBeWritten(t *testing.T) { + var payload codeupipe.Payload + payload.Insert("a", 1.0) + same(t, payload.ToDict(), map[string]any{"a": 1.0}) + same(t, (&codeupipe.Payload{}).Has("a"), false) + same(t, (&codeupipe.Payload{}).Copy().Insert("a", 1.0).ToDict(), map[string]any{"a": 1.0}) +} diff --git a/packages/codeupipe-go/pipeline.go b/packages/codeupipe-go/pipeline.go new file mode 100644 index 0000000..42ed63e --- /dev/null +++ b/packages/codeupipe-go/pipeline.go @@ -0,0 +1,288 @@ +package codeupipe + +import ( + "context" + "strconv" + "sync" + "time" +) + +// PipelineOf runs steps in order on data of type P. Pipeline is the form for +// a Payload; see PipelineData for a type of your own. +// +// Filters change the data, Valves gate a Filter, Taps read, Hooks see the +// lifecycle, and State records what ran. Add steps with AddFilter, +// AddValve, AddTap and AddPipeline, add Hooks with UseHook, and run with +// Run. +// +// A Pipeline can be run from many goroutines at the same time: each run +// writes into a State of its own. Add the steps before the first run. Adding +// a step while a run is in progress is not supported. +type PipelineOf[P PipelineData[P]] struct { + label string + steps []step[P] + uses map[string]int + hooks []HookOf[P] + timing bool + lineage bool + + runs sync.Mutex // guards started and last + // The number of runs that have started; the number of a run is its ticket. + started uint64 + // The State of the run that started last, once it has ended. + last *State +} + +// Pipeline runs steps in order on a Payload. +type Pipeline = PipelineOf[*Payload] + +// step is one step of a pipeline, with its label: a Filter, a Valve, a Tap +// or a nested pipeline. One of the four is set. +type step[P PipelineData[P]] struct { + label string + filter FilterOf[P] + valve *valve[P] + tap TapOf[P] + pipeline *PipelineOf[P] +} + +// NewPipeline gives a pipeline with no steps. Nested, its label is +// "Pipeline". A pipeline type of your own is a NamedPipeline. +func NewPipeline() *Pipeline { + return NewPipelineOf[*Payload]() +} + +// NewPipelineOf is NewPipeline for a pipeline that carries P. +func NewPipelineOf[P PipelineData[P]]() *PipelineOf[P] { + return &PipelineOf[P]{label: "Pipeline"} +} + +// State gives the State of the run that started last, once it has ended. +// +// A State belongs to one run, and this is for a caller with one run at a +// time. A caller with more than one run at a time gives each run its own +// State (RunWithState). Before the first run has ended, the State is empty. +func (p *PipelineOf[P]) State() *State { + p.runs.Lock() + defer p.runs.Unlock() + if p.last == nil { + return NewState() + } + return p.last +} + +// Labels gives the label of each step, in order. +func (p *PipelineOf[P]) Labels() []string { + labels := make([]string, len(p.steps)) + for i := range p.steps { + labels[i] = p.steps[i].label + } + return labels +} + +// AddFilter adds a Filter. +func (p *PipelineOf[P]) AddFilter(filter FilterOf[P]) { + if filter == nil { + panic("codeupipe: AddFilter needs a Filter, a value with Call(ctx, payload). It was given nil") + } + p.push(labelOf(filter), step[P]{filter: filter}) +} + +// AddValve adds a Valve: filter runs only when predicate.Test is true. +// +// The condition comes first, then the Filter that it controls. When the +// test is false the data passes through unchanged and the step is recorded +// as skipped. The step has the label of its Filter. +func (p *PipelineOf[P]) AddValve(predicate PredicateOf[P], filter FilterOf[P]) { + if predicate == nil { + panic("codeupipe: AddValve needs a Predicate, a value with Test(ctx, payload). It was given nil") + } + if filter == nil { + panic("codeupipe: AddValve needs a Filter, a value with Call(ctx, payload). It was given nil") + } + p.push(labelOf(filter), step[P]{valve: &valve[P]{predicate: predicate, filter: filter}}) +} + +// AddTap adds a Tap. +func (p *PipelineOf[P]) AddTap(tap TapOf[P]) { + if tap == nil { + panic("codeupipe: AddTap needs a Tap, a value with Observe(ctx, payload). It was given nil") + } + p.push(labelOf(tap), step[P]{tap: tap}) +} + +// AddPipeline nests a Pipeline as one step of this Pipeline. Its label is +// the name of its NamedPipeline type, or "Pipeline". +func (p *PipelineOf[P]) AddPipeline(pipeline *PipelineOf[P]) { + if pipeline == nil { + panic("codeupipe: AddPipeline needs a Pipeline. It was given nil") + } + p.push(pipeline.label, step[P]{pipeline: pipeline}) +} + +// UseHook adds a Hook. +func (p *PipelineOf[P]) UseHook(hook HookOf[P]) { + if hook == nil { + panic("codeupipe: UseHook needs a Hook, a value with Before, After and OnError. It was given nil") + } + p.hooks = append(p.hooks, hook) +} + +// Observe turns timing (the time of each step, in State) and lineage (labels +// on the data) on or off. Both are off by default. A type of your own keeps +// lineage only if it has Stamp(label string). +func (p *PipelineOf[P]) Observe(timing, lineage bool) { + p.timing = timing + p.lineage = lineage +} + +// push adds a step, with the label base, numbered from its second use in +// this pipeline: "AddFilter", "AddFilter#2". +func (p *PipelineOf[P]) push(base string, s step[P]) { + if p.uses == nil { + p.uses = make(map[string]int) + } + p.uses[base]++ + s.label = base + if count := p.uses[base]; count > 1 { + s.label = base + "#" + strconv.Itoa(count) + } + p.steps = append(p.steps, s) +} + +// Run runs the steps in order on a copy of input, and returns the result. +// The caller's data is not changed. +// +// A Filter failure stops the run. The Hooks see it, and Run returns that +// same error. A Tap failure is recorded in State and the run goes on. The +// run writes into a new State, which State gives once the run has ended. +// +// The Pipeline gives ctx to each step and does nothing else with it: a +// cancelled ctx does not stop the run between steps. +func (p *PipelineOf[P]) Run(ctx context.Context, input P) (P, error) { + return p.RunWithState(ctx, input, NewState()) +} + +// RunWithState is Run, writing into state, which the caller gives. A caller +// with more than one run at a time gives each run its own State. +func (p *PipelineOf[P]) RunWithState(ctx context.Context, input P, state *State) (P, error) { + if state == nil { + panic("codeupipe: RunWithState needs a State. NewState makes one") + } + if state.executed == nil { + // Room for every step, so that the list grows once and not at each step. + state.executed = make([]string, 0, len(p.steps)) + } + ticket := p.start() + payload, err := p.runSteps(ctx, input.Copy(), state) + p.end(ticket, state) + if err != nil { + var none P + return none, err + } + return payload, nil +} + +// start gives a run its ticket. +func (p *PipelineOf[P]) start() uint64 { + p.runs.Lock() + defer p.runs.Unlock() + p.started++ + return p.started +} + +// end keeps state as the State of the pipeline, unless a run that started +// later has ended already (a run inside this run, or in another goroutine). +func (p *PipelineOf[P]) end(ticket uint64, state *State) { + p.runs.Lock() + defer p.runs.Unlock() + if ticket == p.started { + p.last = state + } +} + +func (p *PipelineOf[P]) runSteps(ctx context.Context, payload P, state *State) (P, error) { + for _, hook := range p.hooks { + hook.Before(ctx, "", payload) + } + + for i := range p.steps { + s := &p.steps[i] + if s.tap != nil { + if err := s.tap.Observe(ctx, payload); err != nil { + state.recordError(s.label, err) + } + state.markExecuted(s.label) + continue + } + + var started time.Time + if p.timing { + started = time.Now() + } + for _, hook := range p.hooks { + hook.Before(ctx, s.label, payload) + } + + ran, err := s.run(ctx, &payload) + if err != nil { + if p.timing { + state.recordTiming(s.label, time.Since(started)) + } + // The failing step first, then the pipeline: OnError fires at + // both levels, as Before and After do. + for _, hook := range p.hooks { + hook.OnError(ctx, s.label, err, payload) + } + for _, hook := range p.hooks { + hook.OnError(ctx, "", err, payload) + } + return payload, err + } + if ran { + state.markExecuted(s.label) + } else { + state.markSkipped(s.label) + } + + for _, hook := range p.hooks { + hook.After(ctx, s.label, payload) + } + if p.timing { + state.recordTiming(s.label, time.Since(started)) + } + if p.lineage { + if stamps, ok := any(payload).(stamper); ok { + stamps.Stamp(s.label) + } + } + } + + for _, hook := range p.hooks { + hook.After(ctx, "", payload) + } + return payload, nil +} + +// run runs a step that Hooks see. It is false for a Valve that skipped its +// Filter. +func (s *step[P]) run(ctx context.Context, payload *P) (bool, error) { + switch { + case s.filter != nil: + return true, s.filter.Call(ctx, *payload) + case s.valve != nil: + if !s.valve.predicate.Test(ctx, *payload) { + return false, nil + } + return true, s.valve.filter.Call(ctx, *payload) + default: + // A run of its own, with its own State: a failure inside leaves this + // data as it was. + out, err := s.pipeline.Run(ctx, *payload) + if err != nil { + return true, err + } + *payload = out + return true, nil + } +} diff --git a/packages/codeupipe-go/pipeline_data.go b/packages/codeupipe-go/pipeline_data.go new file mode 100644 index 0000000..da7d78f --- /dev/null +++ b/packages/codeupipe-go/pipeline_data.go @@ -0,0 +1,35 @@ +package codeupipe + +// PipelineData is what a pipeline can carry from step to step. +// +// Payload is the default: a map that any developer can read, and that every +// runtime of codeupipe has. A type of your own can take its place, such as +// the struct that your steps would use anyway: +// +// type Order struct { +// Price float64 +// Qty float64 +// Bulk bool +// } +// +// func (o *Order) Copy() *Order { +// copied := *o +// return &copied +// } +// +// pipeline := codeupipe.NewPipelineOf[*Order]() +// +// The Pipeline asks one thing of it, Copy, for the copy that a run works on. +// The Filters, Predicates, Taps and Hooks of that pipeline then take the +// type, and a step for another type does not compile. +// +// Lineage (Pipeline.Observe) is kept only by a type that also has +// Stamp(label string), as Payload has. For any other type it does nothing. +type PipelineData[P any] interface { + Copy() P +} + +// stamper is a type that keeps lineage. +type stamper interface { + Stamp(label string) +} diff --git a/packages/codeupipe-go/pipeline_data_test.go b/packages/codeupipe-go/pipeline_data_test.go new file mode 100644 index 0000000..00a2efe --- /dev/null +++ b/packages/codeupipe-go/pipeline_data_test.go @@ -0,0 +1,183 @@ +package codeupipe_test + +// A pipeline over a type of your own. Payload is the default, a map that any +// developer can read; a struct can take its place. The Pipeline asks one +// thing of it, Copy, for the copy that a run works on. Lineage is kept only +// by a type that has Stamp(label); for any other type it does nothing. + +import ( + "context" + "errors" + "math" + "strconv" + "testing" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +type Order struct { + Price float64 + Qty float64 + Bulk bool +} + +func (o *Order) Copy() *Order { + copied := *o + return &copied +} + +type TracedOrder struct { + Order + Steps []string +} + +func (o *TracedOrder) Copy() *TracedOrder { + return &TracedOrder{Order: o.Order, Steps: append([]string{}, o.Steps...)} +} + +func (o *TracedOrder) Stamp(label string) { + o.Steps = append(o.Steps, label) +} + +type OrderDiscountFilter struct{} + +func (OrderDiscountFilter) Call(_ context.Context, order *Order) error { + order.Price = math.Round(order.Price*90) / 100 + return nil +} + +type OrderBulkFilter struct{} + +func (OrderBulkFilter) Call(_ context.Context, order *Order) error { + order.Bulk = true + return nil +} + +type RefuseFreeFilter struct{} + +func (RefuseFreeFilter) Call(_ context.Context, order *Order) error { + if order.Price <= 0 { + return errors.New("an order must cost something") + } + return nil +} + +type PassTracedFilter struct{} + +func (PassTracedFilter) Call(context.Context, *TracedOrder) error { + return nil +} + +type BulkPredicate struct{} + +func (BulkPredicate) Test(_ context.Context, order *Order) bool { + return order.Qty >= 10 +} + +type SeePriceTap struct { + seen []float64 +} + +func (t *SeePriceTap) Observe(_ context.Context, order *Order) error { + t.seen = append(t.seen, order.Price) + return nil +} + +type RecordOrderHook struct { + codeupipe.HookBaseOf[*Order] + events []string +} + +func (h *RecordOrderHook) Before(_ context.Context, step string, _ *Order) { + h.events = append(h.events, "before:"+step) +} + +func (h *RecordOrderHook) OnError(_ context.Context, step string, _ error, order *Order) { + h.events = append(h.events, "error:"+step+":"+strconv.FormatFloat(order.Price, 'g', -1, 64)) +} + +type OrderPricingPipeline struct{} + +func (OrderPricingPipeline) AddSteps(pipeline *codeupipe.PipelineOf[*Order]) { + pipeline.AddFilter(OrderDiscountFilter{}) + pipeline.AddValve(BulkPredicate{}, OrderBulkFilter{}) +} + +func TestEveryPrimitiveTakesTheType(t *testing.T) { + tap := &SeePriceTap{} + hook := &RecordOrderHook{} + pipeline := codeupipe.NewNamedPipelineOf[*Order](OrderPricingPipeline{}) + pipeline.AddTap(tap) + pipeline.UseHook(hook) + given := &Order{Price: 100, Qty: 12} + out, err := pipeline.Run(context.Background(), given) + same(t, err, nil) + same(t, out, &Order{Price: 90, Qty: 12, Bulk: true}) + same(t, given, &Order{Price: 100, Qty: 12}) // the caller's value is not changed + same(t, tap.seen, []float64{90}) + same(t, pipeline.State().Executed(), []string{"OrderDiscountFilter", "OrderBulkFilter", "SeePriceTap"}) + same(t, hook.events, []string{"before:", "before:OrderDiscountFilter", "before:OrderBulkFilter"}) + _, err = pipeline.Run(context.Background(), &Order{Price: 100, Qty: 2}) + same(t, err, nil) + same(t, pipeline.State().Skipped(), []string{"OrderBulkFilter"}) +} + +func TestANestedPipelineTakesTheType(t *testing.T) { + pipeline := codeupipe.NewPipelineOf[*Order]() + pipeline.AddPipeline(codeupipe.NewNamedPipelineOf[*Order](OrderPricingPipeline{})) + pipeline.AddFilter(RefuseFreeFilter{}) + out, err := pipeline.Run(context.Background(), &Order{Price: 50, Qty: 10}) + same(t, err, nil) + same(t, out, &Order{Price: 45, Qty: 10, Bulk: true}) + same(t, pipeline.State().Executed(), []string{"OrderPricingPipeline", "RefuseFreeFilter"}) +} + +func TestAFailureIsTheSameAsWithAPayload(t *testing.T) { + hook := &RecordOrderHook{} + pipeline := codeupipe.NewPipelineOf[*Order]() + pipeline.AddFilter(OrderDiscountFilter{}) + pipeline.AddFilter(RefuseFreeFilter{}) + pipeline.UseHook(hook) + out, err := pipeline.Run(context.Background(), &Order{Price: 0, Qty: 1}) + same(t, out, (*Order)(nil)) + same(t, err.Error(), "an order must cost something") + same(t, pipeline.State().Executed(), []string{"OrderDiscountFilter"}) + same(t, hook.events[len(hook.events)-2:], []string{"error:RefuseFreeFilter:0", "error::0"}) +} + +func TestLineageDoesNothingForATypeWithoutStamp(t *testing.T) { + pipeline := codeupipe.NewNamedPipelineOf[*Order](OrderPricingPipeline{}) + pipeline.Observe(false, true) + out, err := pipeline.Run(context.Background(), &Order{Price: 100, Qty: 12}) + same(t, err, nil) + same(t, out, &Order{Price: 90, Qty: 12, Bulk: true}) + same(t, pipeline.State().Executed(), []string{"OrderDiscountFilter", "OrderBulkFilter"}) +} + +func TestLineageIsKeptByATypeWithStamp(t *testing.T) { + pipeline := codeupipe.NewPipelineOf[*TracedOrder]() + pipeline.AddFilter(PassTracedFilter{}) + pipeline.Observe(false, true) + given := &TracedOrder{Order: Order{Price: 100, Qty: 12}} + out, err := pipeline.Run(context.Background(), given) + same(t, err, nil) + same(t, out.Steps, []string{"PassTracedFilter"}) + same(t, given.Steps, []string(nil)) +} + +// MarkRanFilter is a Filter of a Payload. +type MarkRanFilter struct{} + +func (MarkRanFilter) Call(_ context.Context, payload *codeupipe.Payload) error { + payload.Insert("ran", true) + return nil +} + +func TestAPayloadPipelineIsStillTheDefault(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(MarkRanFilter{}) + same(t, run(t, pipeline, nil).ToDict(), map[string]any{"ran": true}) +} + +// A pipeline carries one type: a step for another type does not compile. +// TestWhatTheContractRefusesDoesNotCompile checks it. diff --git a/packages/codeupipe-go/pipeline_test.go b/packages/codeupipe-go/pipeline_test.go new file mode 100644 index 0000000..3be62a9 --- /dev/null +++ b/packages/codeupipe-go/pipeline_test.go @@ -0,0 +1,382 @@ +package codeupipe_test + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +func run(t *testing.T, pipeline *codeupipe.Pipeline, data map[string]any) *codeupipe.Payload { + t.Helper() + out, err := pipeline.Run(context.Background(), codeupipe.NewPayload(data)) + if err != nil { + t.Fatalf("the run failed: %v", err) + } + return out +} + +func TestAnEmptyPipelineReturnsTheDataItWasGiven(t *testing.T) { + out := run(t, codeupipe.NewPipeline(), map[string]any{"a": 1.0}) + same(t, out.ToDict(), map[string]any{"a": 1.0}) +} + +func TestStepsRunInOrderAndStateRecordsThem(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(AddFilter{}) + out := run(t, pipeline, nil) + same(t, out.Get("n"), 2.0) + same(t, pipeline.State().Executed(), []string{"AddFilter", "AddFilter#2"}) + same(t, pipeline.State().Skipped(), []string{}) + same(t, pipeline.State().Errors(), []codeupipe.StepFailure{}) +} + +func TestRunWorksOnACopyTheCallersPayloadIsNotChanged(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + given := codeupipe.NewPayload(map[string]any{"n": 1.0}) + out, err := pipeline.Run(context.Background(), given) + same(t, err, nil) + same(t, given.Get("n"), 1.0) + same(t, out.Get("n"), 2.0) +} + +func TestAValveRunsItsFilterWhenThePredicateIsTrue(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddValve(IsOnPredicate{}, AddFilter{}) + out := run(t, pipeline, map[string]any{"on": true}) + same(t, out.Get("n"), 1.0) + same(t, pipeline.State().Executed(), []string{"AddFilter"}) + same(t, pipeline.State().Skipped(), []string{}) +} + +func TestAValveSkipsItsFilterWhenThePredicateIsFalse(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddValve(IsOnPredicate{}, AddFilter{}) + out := run(t, pipeline, map[string]any{"on": false}) + same(t, out.Has("n"), false) + same(t, pipeline.State().Executed(), []string{}) + same(t, pipeline.State().Skipped(), []string{"AddFilter"}) +} + +func TestAFilterFailureStopsTheRunAndStateKeepsTheStepsBeforeIt(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(FailFilter{}) + pipeline.AddFilter(AddFilter{}) + out, err := pipeline.Run(context.Background(), codeupipe.NewPayload(nil)) + same(t, out, (*codeupipe.Payload)(nil)) + same(t, err, errBoom) // the error itself, so errors.Is works + same(t, pipeline.State().Executed(), []string{"AddFilter"}) +} + +func TestATapReadsThePayloadAndCountsAsExecuted(t *testing.T) { + tap := &SeeTap{} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddTap(tap) + run(t, pipeline, nil) + same(t, tap.seen, []any{1.0}) + same(t, pipeline.State().Executed(), []string{"AddFilter", "SeeTap"}) +} + +func TestATapFailureIsRecordedAndDoesNotStopTheRun(t *testing.T) { + hook := &RecordHook{} + pipeline := codeupipe.NewPipeline() + pipeline.AddTap(FailTap{}) + pipeline.AddTap(FailTap{}) + pipeline.AddFilter(AddFilter{}) + pipeline.UseHook(hook) + out := run(t, pipeline, nil) + same(t, out.Get("n"), 1.0) + state := pipeline.State() + same(t, state.Executed(), []string{"FailTap", "FailTap#2", "AddFilter"}) + same(t, len(state.Errors()), 2) + same(t, state.Errors()[0].Label, "FailTap") + same(t, state.Errors()[1].Label, "FailTap#2") + same(t, state.Errors()[0].Err.Error(), "sink is down") + same(t, hook.events, []string{"before:", "before:AddFilter", "after:AddFilter", "after:"}) +} + +func TestANestedPipelineIsOneStep(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddPipeline(codeupipe.NewNamedPipeline(InnerPipeline{})) + pipeline.AddFilter(AddFilter{}) + out := run(t, pipeline, nil) + same(t, out.Get("n"), 3.0) + same(t, pipeline.State().Executed(), []string{"InnerPipeline", "AddFilter"}) +} + +func TestHooksSeeThePipelineAndEachFilterOrValveInOrder(t *testing.T) { + hook := &RecordHook{} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddValve(IsOnPredicate{}, FailFilter{}) + pipeline.AddTap(&SeeTap{}) + pipeline.UseHook(hook) + run(t, pipeline, nil) + same(t, hook.events, []string{ + "before:", "before:AddFilter", "after:AddFilter", + "before:FailFilter", "after:FailFilter", "after:", + }) +} + +func TestOnAFilterFailureAHookGetsTheStepThenThePipelineAndNoAfter(t *testing.T) { + hook := &RecordHook{} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(FailFilter{}) + pipeline.UseHook(hook) + _, err := pipeline.Run(context.Background(), codeupipe.NewPayload(nil)) + same(t, err, errBoom) + same(t, hook.events, []string{ + "before:", "before:AddFilter", "after:AddFilter", + "before:FailFilter", "error:FailFilter", "error:", + }) +} + +// KeepErrorHook keeps each error, and the Payload that OnError sees. +type KeepErrorHook struct { + codeupipe.HookBase + errors []error + payloads []map[string]any +} + +func (h *KeepErrorHook) OnError(_ context.Context, _ string, err error, payload *codeupipe.Payload) { + h.errors = append(h.errors, err) + h.payloads = append(h.payloads, payload.ToDict()) +} + +// WriteThenFailFilter writes, then fails. +type WriteThenFailFilter struct{} + +func (WriteThenFailFilter) Call(_ context.Context, payload *codeupipe.Payload) error { + payload.Insert("written", true) + return errBoom +} + +func TestAHookGetsTheErrorItselfAndThePayloadAsTheFailingStepLeftIt(t *testing.T) { + hook := &KeepErrorHook{} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(WriteThenFailFilter{}) + pipeline.UseHook(hook) + _, err := pipeline.Run(context.Background(), codeupipe.NewPayload(nil)) + same(t, errors.Is(err, errBoom), true) + same(t, hook.errors, []error{errBoom, errBoom}) + same(t, hook.payloads, []map[string]any{{"written": true}, {"written": true}}) +} + +func TestAFailureInANestedPipelineLeavesThePayloadOfTheOuterOneAsItWas(t *testing.T) { + inner := codeupipe.NewPipeline() + inner.AddFilter(WriteThenFailFilter{}) + hook := &KeepErrorHook{} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddPipeline(inner) + pipeline.UseHook(hook) + _, err := pipeline.Run(context.Background(), codeupipe.NewPayload(nil)) + same(t, err, errBoom) + same(t, hook.payloads, []map[string]any{{"n": 1.0}, {"n": 1.0}}) + same(t, pipeline.State().Executed(), []string{"AddFilter"}) +} + +func TestTimingAndLineageAreOffByDefault(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + out := run(t, pipeline, nil) + same(t, len(pipeline.State().Timings()), 0) + same(t, out.Lineage(), []string{}) +} + +func TestObserveTurnsOnTimingAndLineage(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddTap(&SeeTap{}) + pipeline.AddValve(IsOnPredicate{}, FailFilter{}) + pipeline.Observe(true, true) + out := run(t, pipeline, nil) + timings := pipeline.State().Timings() + same(t, len(timings), 2) // a Tap is not timed + same(t, timings["AddFilter"] >= 0, true) + same(t, timings["FailFilter"] >= 0, true) + same(t, out.Lineage(), []string{"AddFilter", "FailFilter"}) +} + +func TestEachRunStartsWithANewState(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddValve(IsOnPredicate{}, AddFilter{}) + run(t, pipeline, map[string]any{"on": false}) + run(t, pipeline, map[string]any{"on": true}) + same(t, pipeline.State().Executed(), []string{"AddFilter"}) + same(t, pipeline.State().Skipped(), []string{}) +} + +func TestBeforeAnyRunTheStateIsEmpty(t *testing.T) { + pipeline := codeupipe.NewPipeline() + same(t, pipeline.State().Executed(), []string{}) +} + +// -- context ---------------------------------------------------------------- + +type requestKey struct{} + +// SeeRequestFilter, SeeRequestPredicate, SeeRequestTap and SeeRequestHook +// each keep the request of the context that they get. +type SeeRequestFilter struct{ seen *[]any } + +func (f SeeRequestFilter) Call(ctx context.Context, _ *codeupipe.Payload) error { + *f.seen = append(*f.seen, ctx.Value(requestKey{})) + return nil +} + +type SeeRequestPredicate struct{ seen *[]any } + +func (q SeeRequestPredicate) Test(ctx context.Context, _ *codeupipe.Payload) bool { + *q.seen = append(*q.seen, ctx.Value(requestKey{})) + return true +} + +type SeeRequestTap struct{ seen *[]any } + +func (t SeeRequestTap) Observe(ctx context.Context, _ *codeupipe.Payload) error { + *t.seen = append(*t.seen, ctx.Value(requestKey{})) + return nil +} + +type SeeRequestHook struct { + codeupipe.HookBase + seen *[]any +} + +func (h SeeRequestHook) Before(ctx context.Context, _ string, _ *codeupipe.Payload) { + *h.seen = append(*h.seen, ctx.Value(requestKey{})) +} + +func TestThePipelineGivesTheContextToEachStep(t *testing.T) { + var seen []any + inner := codeupipe.NewPipeline() + inner.AddFilter(SeeRequestFilter{&seen}) + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(SeeRequestFilter{&seen}) + pipeline.AddValve(SeeRequestPredicate{&seen}, SeeRequestFilter{&seen}) + pipeline.AddTap(SeeRequestTap{&seen}) + pipeline.AddPipeline(inner) + pipeline.UseHook(SeeRequestHook{seen: &seen}) + ctx := context.WithValue(context.Background(), requestKey{}, "r-7") + _, err := pipeline.Run(ctx, codeupipe.NewPayload(nil)) + same(t, err, nil) + // The Hook before the pipeline and before each of its 3 gated steps, the + // 2 Filters, the Predicate, the Tap and the Filter of the nested pipeline. + same(t, len(seen), 9) + for _, request := range seen { + same(t, request, "r-7") + } +} + +func TestACancelledContextDoesNotStopTheRun(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(AddFilter{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out, err := pipeline.Run(ctx, codeupipe.NewPayload(nil)) + same(t, err, nil) + same(t, out.Get("n"), 2.0) +} + +// -- a State belongs to one run --------------------------------------------- + +func TestRunsInManyGoroutinesDoNotWriteIntoTheSameState(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(AddFilter{}) + pipeline.AddTap(FailTap{}) + var wait sync.WaitGroup + outs := make([]any, 50) + for i := range outs { + wait.Add(1) + go func() { + defer wait.Done() + out, err := pipeline.Run(context.Background(), codeupipe.NewPayload(nil)) + if err == nil { + outs[i] = out.Get("n") + } + }() + } + wait.Wait() + for _, n := range outs { + same(t, n, 2.0) + } + state := pipeline.State() + same(t, state.Executed(), []string{"AddFilter", "AddFilter#2", "FailTap"}) + same(t, len(state.Errors()), 1) +} + +// WaitFilter keeps the run of "A" waiting until the test lets it go on. +type WaitFilter struct { + started chan struct{} + release chan struct{} +} + +func (f WaitFilter) Call(_ context.Context, payload *codeupipe.Payload) error { + if payload.Get("name") == "A" { + close(f.started) + <-f.release + } + return nil +} + +// FailForBFilter fails for "B". +type FailForBFilter struct{} + +func (FailForBFilter) Call(_ context.Context, payload *codeupipe.Payload) error { + if payload.Get("name") == "B" { + return errors.New("B fails") + } + return nil +} + +func TestEachOfTwoRunsAtTheSameTimeWritesIntoTheStateThatItsCallerGave(t *testing.T) { + wait := WaitFilter{started: make(chan struct{}), release: make(chan struct{})} + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(wait) + pipeline.AddFilter(FailForBFilter{}) + a, b := codeupipe.NewState(), codeupipe.NewState() + // A starts first and ends last. B starts second, fails, and ends first. + done := make(chan error) + go func() { + _, err := pipeline.RunWithState(context.Background(), codeupipe.NewPayload(map[string]any{"name": "A"}), a) + done <- err + }() + <-wait.started + _, err := pipeline.RunWithState(context.Background(), codeupipe.NewPayload(map[string]any{"name": "B"}), b) + same(t, err.Error(), "B fails") + close(wait.release) + same(t, <-done, nil) + same(t, a.Executed(), []string{"WaitFilter", "FailForBFilter"}) + same(t, b.Executed(), []string{"WaitFilter"}) + same(t, pipeline.State() == b, true) // the State of the run that started last +} + +func TestWithNoStateFromTheCallerARunMakesANewOne(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(FailForBFilter{}) + given := codeupipe.NewState() + _, err := pipeline.RunWithState(context.Background(), codeupipe.NewPayload(map[string]any{"name": "A"}), given) + same(t, err, nil) + same(t, pipeline.State() == given, true) + run(t, pipeline, map[string]any{"name": "A"}) + same(t, pipeline.State() == given, false) + same(t, given.Executed(), []string{"FailForBFilter"}) + same(t, pipeline.State().Executed(), []string{"FailForBFilter"}) +} + +func TestRunWithStateNeedsAState(t *testing.T) { + defer func() { + same(t, recover(), "codeupipe: RunWithState needs a State. NewState makes one") + }() + _, _ = codeupipe.NewPipeline().RunWithState(context.Background(), codeupipe.NewPayload(nil), nil) +} diff --git a/packages/codeupipe-go/predicate.go b/packages/codeupipe-go/predicate.go new file mode 100644 index 0000000..ab08db7 --- /dev/null +++ b/packages/codeupipe-go/predicate.go @@ -0,0 +1,24 @@ +package codeupipe + +import "context" + +// PredicateOf asks a yes-or-no question about the data of a pipeline that +// carries P. It reads the data and does not write to it. +type PredicateOf[P any] interface { + Test(ctx context.Context, payload P) bool +} + +// Predicate asks a yes-or-no question about the Payload. Its settings (a +// threshold, a flag) are fields of the value, and one Predicate can be used +// in many Valves. +// +// type AtLeastPredicate struct { +// Key string +// Minimum float64 +// } +// +// func (q AtLeastPredicate) Test(ctx context.Context, payload *codeupipe.Payload) bool { +// value, _ := payload.Get(q.Key).(float64) +// return value >= q.Minimum +// } +type Predicate = PredicateOf[*Payload] diff --git a/packages/codeupipe-go/refused_test.go b/packages/codeupipe-go/refused_test.go new file mode 100644 index 0000000..aca30aa --- /dev/null +++ b/packages/codeupipe-go/refused_test.go @@ -0,0 +1,119 @@ +package codeupipe_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// What the contract refuses with a TypeError when a step is added is a +// compile error in Go. Each shape below is built as a program of its own, +// and the build must fail with the error that names the cause. + +const refusedPrelude = `package main + +import ( + "context" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +type Order struct{ Price float64 } + +func (o *Order) Copy() *Order { copied := *o; return &copied } + +type OrderFilter struct{} + +func (OrderFilter) Call(context.Context, *Order) error { return nil } + +type MarkFilter struct{} + +func (MarkFilter) Call(context.Context, *codeupipe.Payload) error { return nil } + +type IsOnPredicate struct{} + +func (IsOnPredicate) Test(context.Context, *codeupipe.Payload) bool { return true } + +func main() { + ctx := context.Background() + pipeline := codeupipe.NewPipeline() + _, _ = ctx, pipeline +` + +var refusedShapes = []struct { + name, code, error string +}{ + {"a function as a step", + `pipeline.AddFilter(func(context.Context, *codeupipe.Payload) error { return nil })`, + "does not implement codeupipe.FilterOf[*codeupipe.Payload] (missing method Call)"}, + {"a Tap that is a Filter", + `pipeline.AddTap(MarkFilter{})`, + "does not implement codeupipe.TapOf[*codeupipe.Payload] (missing method Observe)"}, + {"the arguments of AddValve in the wrong order", + `pipeline.AddValve(MarkFilter{}, IsOnPredicate{})`, + "does not implement codeupipe.PredicateOf[*codeupipe.Payload] (missing method Test)"}, + {"a step for another type", + `pipeline.AddFilter(OrderFilter{})`, + "does not implement codeupipe.FilterOf[*codeupipe.Payload] (wrong type for method Call)"}, + {"a run of another type", + `codeupipe.NewPipelineOf[*Order]().Run(ctx, codeupipe.NewPayload(nil))`, + "cannot use codeupipe.NewPayload(nil)"}, + {"a type with no Copy of its own type", + `codeupipe.NewPipelineOf[Order]()`, + "Order does not satisfy codeupipe.PipelineData[Order]"}, +} + +func TestWhatTheContractRefusesDoesNotCompile(t *testing.T) { + if testing.Short() { + t.Skip("builds one program for each shape") + } + tool, err := exec.LookPath("go") + if err != nil { + t.Skip("no go command") + } + here, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + module := t.TempDir() + write(t, filepath.Join(module, "go.mod"), "module refused\n\ngo 1.22\n\n"+ + "require github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go v0.0.0\n\n"+ + "replace github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go => "+here+"\n") + build := func(t *testing.T, name, code string) ([]byte, error) { + t.Helper() + dir := filepath.Join(module, name) + write(t, filepath.Join(dir, "main.go"), refusedPrelude+"\t"+code+"\n}\n") + command := exec.Command(tool, "build", "-o", os.DevNull, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOFLAGS=-mod=mod", "GOPROXY=off", "GOWORK=off") + return command.CombinedOutput() + } + // The program with the right steps compiles, so a failure below comes + // from the shape and not from the program around it. + if out, err := build(t, "allowed", `pipeline.AddValve(IsOnPredicate{}, MarkFilter{})`); err != nil { + t.Fatalf("the program with the right steps does not compile:\n%s", out) + } + for i, shape := range refusedShapes { + t.Run(shape.name, func(t *testing.T) { + out, err := build(t, "shape"+string(rune('a'+i)), shape.code) + if err == nil { + t.Fatalf("it compiles: %s", shape.code) + } + if !strings.Contains(string(out), shape.error) { + t.Fatalf("the build fails, but not with %q:\n%s", shape.error, out) + } + }) + } +} + +func write(t *testing.T, path, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/packages/codeupipe-go/state.go b/packages/codeupipe-go/state.go new file mode 100644 index 0000000..d372035 --- /dev/null +++ b/packages/codeupipe-go/state.go @@ -0,0 +1,67 @@ +package codeupipe + +import ( + "maps" + "time" +) + +// State records what happened in one run: the steps that executed, the +// steps that a Valve skipped, the errors, and the time of each step when +// timing is on. +// +// A State belongs to one run. Give one to Pipeline.RunWithState, or read the +// State of the run that started last with Pipeline.State. A failed run keeps +// the steps that completed before the failure. +type State struct { + executed []string + skipped []string + errors []StepFailure + timings map[string]time.Duration +} + +// NewState gives an empty State, for a caller that gives the State of a run. +func NewState() *State { + return &State{} +} + +// Executed gives the labels of the steps that ran, in order. +func (s *State) Executed() []string { + return append([]string{}, s.executed...) +} + +// Skipped gives the labels of the steps that a Valve did not run. +func (s *State) Skipped() []string { + return append([]string{}, s.skipped...) +} + +// Errors gives the errors of the run, each with the label of its step. +func (s *State) Errors() []StepFailure { + return append([]StepFailure{}, s.errors...) +} + +// Timings gives the time of each step, by label, when timing is on. +func (s *State) Timings() map[string]time.Duration { + if s.timings == nil { + return map[string]time.Duration{} + } + return maps.Clone(s.timings) +} + +func (s *State) markExecuted(label string) { + s.executed = append(s.executed, label) +} + +func (s *State) markSkipped(label string) { + s.skipped = append(s.skipped, label) +} + +func (s *State) recordError(label string, err error) { + s.errors = append(s.errors, StepFailure{Label: label, Err: err}) +} + +func (s *State) recordTiming(label string, elapsed time.Duration) { + if s.timings == nil { + s.timings = make(map[string]time.Duration) + } + s.timings[label] = elapsed +} diff --git a/packages/codeupipe-go/step_failure.go b/packages/codeupipe-go/step_failure.go new file mode 100644 index 0000000..3930bf4 --- /dev/null +++ b/packages/codeupipe-go/step_failure.go @@ -0,0 +1,8 @@ +package codeupipe + +// StepFailure is an error that State keeps: the label of the step that +// failed, and the error itself. +type StepFailure struct { + Label string + Err error +} diff --git a/packages/codeupipe-go/step_label.go b/packages/codeupipe-go/step_label.go new file mode 100644 index 0000000..8e1f5eb --- /dev/null +++ b/packages/codeupipe-go/step_label.go @@ -0,0 +1,28 @@ +package codeupipe + +import ( + "reflect" + "strings" +) + +// labelOf gives the label of a step: the name of its type. For a pointer it +// is the type that it points to, and type arguments are dropped +// (Wrapper[pkg.Inner] gives Wrapper). A type with no name gives "anonymous". +// State and Hooks report a step by this label. +func labelOf(step any) string { + t := reflect.TypeOf(step) + for t != nil && t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t == nil { + return "anonymous" + } + name := t.Name() + if open := strings.IndexByte(name, '['); open >= 0 { + name = name[:open] + } + if name == "" { + return "anonymous" + } + return name +} diff --git a/packages/codeupipe-go/step_label_test.go b/packages/codeupipe-go/step_label_test.go new file mode 100644 index 0000000..b70dcc7 --- /dev/null +++ b/packages/codeupipe-go/step_label_test.go @@ -0,0 +1,98 @@ +package codeupipe_test + +import ( + "context" + "testing" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// WrapFilter is a generic step: its label has no type arguments. +type WrapFilter[T any] struct{} + +func (WrapFilter[T]) Call(context.Context, *codeupipe.Payload) error { + return nil +} + +func TestTheLabelIsTheNameOfTheType(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddTap(&SeeTap{}) + same(t, pipeline.Labels(), []string{"AddFilter", "SeeTap"}) +} + +func TestTheSecondUseOfATypeGetsANumber(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(&AddFilter{}) + pipeline.AddFilter(AddFilter{}) + same(t, pipeline.Labels(), []string{"AddFilter", "AddFilter#2", "AddFilter#3"}) +} + +func TestAValveHasTheLabelOfItsFilter(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.AddValve(IsOnPredicate{}, AddFilter{}) + same(t, pipeline.Labels(), []string{"AddFilter", "AddFilter#2"}) +} + +func TestANestedPipelineHasTheLabelOfItsType(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddPipeline(codeupipe.NewNamedPipeline(InnerPipeline{})) + pipeline.AddPipeline(codeupipe.NewPipeline()) + same(t, pipeline.Labels(), []string{"InnerPipeline", "Pipeline"}) +} + +func TestTheLabelOfAGenericTypeHasNoTypeArguments(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(WrapFilter[AddFilter]{}) + same(t, pipeline.Labels(), []string{"WrapFilter"}) +} + +func TestATypeDeclaredInAFunctionWorks(t *testing.T) { + type LocalFilter struct{ AddFilter } + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(LocalFilter{}) + same(t, pipeline.Labels(), []string{"LocalFilter"}) +} + +func TestAStepOfATypeWithNoNameHasTheLabelAnonymous(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(struct{ AddFilter }{}) + same(t, pipeline.Labels(), []string{"anonymous"}) +} + +func TestLabelsGivesANewListEachTime(t *testing.T) { + pipeline := codeupipe.NewPipeline() + pipeline.AddFilter(AddFilter{}) + pipeline.Labels()[0] = "x" + same(t, pipeline.Labels(), []string{"AddFilter"}) +} + +// A value with no Call is not a Filter, and AddValve(filter, predicate) does +// not compile: those are checked by the compiler. A nil step compiles, and +// is refused when it is added. +func TestANilStepIsRefusedWhenItIsAdded(t *testing.T) { + refused := func(add func(), message string) { + t.Helper() + defer func() { + t.Helper() + same(t, recover(), message) + }() + add() + } + pipeline := codeupipe.NewPipeline() + refused(func() { pipeline.AddFilter(nil) }, + "codeupipe: AddFilter needs a Filter, a value with Call(ctx, payload). It was given nil") + refused(func() { pipeline.AddValve(nil, AddFilter{}) }, + "codeupipe: AddValve needs a Predicate, a value with Test(ctx, payload). It was given nil") + refused(func() { pipeline.AddValve(IsOnPredicate{}, nil) }, + "codeupipe: AddValve needs a Filter, a value with Call(ctx, payload). It was given nil") + refused(func() { pipeline.AddTap(nil) }, + "codeupipe: AddTap needs a Tap, a value with Observe(ctx, payload). It was given nil") + refused(func() { pipeline.AddPipeline(nil) }, + "codeupipe: AddPipeline needs a Pipeline. It was given nil") + refused(func() { pipeline.UseHook(nil) }, + "codeupipe: UseHook needs a Hook, a value with Before, After and OnError. It was given nil") + same(t, pipeline.Labels(), []string{}) +} diff --git a/packages/codeupipe-go/steps_test.go b/packages/codeupipe-go/steps_test.go new file mode 100644 index 0000000..623d0b6 --- /dev/null +++ b/packages/codeupipe-go/steps_test.go @@ -0,0 +1,78 @@ +package codeupipe_test + +import ( + "context" + "errors" + + "github.com/orchestrate-solutions/codeupipe/packages/codeupipe-go" +) + +// The steps that the tests of the Pipeline and of the labels share. + +var errBoom = errors.New("boom") + +// AddFilter adds 1 to "n". +type AddFilter struct{} + +func (AddFilter) Call(_ context.Context, payload *codeupipe.Payload) error { + n, _ := payload.Get("n").(float64) + payload.Insert("n", n+1) + return nil +} + +// FailFilter fails with errBoom. +type FailFilter struct{} + +func (FailFilter) Call(context.Context, *codeupipe.Payload) error { + return errBoom +} + +// IsOnPredicate is true when "on" is true. +type IsOnPredicate struct{} + +func (IsOnPredicate) Test(_ context.Context, payload *codeupipe.Payload) bool { + return payload.Get("on") == true +} + +// SeeTap keeps each value of "n" that it sees. +type SeeTap struct { + seen []any +} + +func (t *SeeTap) Observe(_ context.Context, payload *codeupipe.Payload) error { + t.seen = append(t.seen, payload.Get("n")) + return nil +} + +// FailTap fails. +type FailTap struct{} + +func (FailTap) Observe(context.Context, *codeupipe.Payload) error { + return errors.New("sink is down") +} + +// InnerPipeline adds 1 to "n" two times. +type InnerPipeline struct{} + +func (InnerPipeline) AddSteps(pipeline *codeupipe.Pipeline) { + pipeline.AddFilter(AddFilter{}) + pipeline.AddFilter(AddFilter{}) +} + +// RecordHook keeps each event as "before:", "after:" or +// "error:". The step of the pipeline itself is "". +type RecordHook struct { + events []string +} + +func (h *RecordHook) Before(_ context.Context, step string, _ *codeupipe.Payload) { + h.events = append(h.events, "before:"+step) +} + +func (h *RecordHook) After(_ context.Context, step string, _ *codeupipe.Payload) { + h.events = append(h.events, "after:"+step) +} + +func (h *RecordHook) OnError(_ context.Context, step string, _ error, _ *codeupipe.Payload) { + h.events = append(h.events, "error:"+step) +} diff --git a/packages/codeupipe-go/tap.go b/packages/codeupipe-go/tap.go new file mode 100644 index 0000000..af0e9b6 --- /dev/null +++ b/packages/codeupipe-go/tap.go @@ -0,0 +1,13 @@ +package codeupipe + +import "context" + +// TapOf reads the data of a pipeline that carries P as it goes by. It does +// not write to it. A failure never stops the run: State records the error +// under the label of the Tap, and Hooks are not told. +type TapOf[P any] interface { + Observe(ctx context.Context, payload P) error +} + +// Tap reads the Payload as it goes by, to log it or to send it somewhere. +type Tap = TapOf[*Payload] diff --git a/packages/codeupipe-go/valve.go b/packages/codeupipe-go/valve.go new file mode 100644 index 0000000..5582212 --- /dev/null +++ b/packages/codeupipe-go/valve.go @@ -0,0 +1,8 @@ +package codeupipe + +// valve joins a Predicate to a Filter: the Filter runs only when the +// Predicate is true. Pipeline.AddValve makes one. It is not a public name. +type valve[P any] struct { + predicate PredicateOf[P] + filter FilterOf[P] +} diff --git a/tests/conformance/test_bench_baseline.py b/tests/conformance/test_bench_baseline.py index 85964f4..a27dc9b 100644 --- a/tests/conformance/test_bench_baseline.py +++ b/tests/conformance/test_bench_baseline.py @@ -66,7 +66,7 @@ def test_the_baseline_file_holds_a_ratio_for_every_number(): baseline = bench.load_baseline() assert baseline, "run: python conformance/bench.py --record" for key, recorded in baseline.items(): - runtime = "rust" if key == "rust" else "python" # the other keys are Python versions + runtime = key if key in ("rust", "go") else "python" # the other keys are Python versions want = {target for target in bench.BENCH_TARGETS if target.split(":")[0] == runtime} assert set(recorded["relative"]) == want, key for target, row in recorded["relative"].items(): @@ -79,6 +79,10 @@ def test_a_rust_number_is_kept_apart_from_the_python_version(): assert bench.baseline_key("rust:run", python="3.12") == "rust" +def test_a_go_number_is_kept_apart_from_the_python_version(): + assert bench.baseline_key("go:run", python="3.12") == "go" + + def test_the_check_compares_each_target_with_its_own_baseline(): baseline = {"3.12": {"relative": {"python:run": {"m": 5.0}, "python:static-inline": {"m": 0.5}}}, "3.9": {"relative": {"python:run": {"m": 9.0}}}, diff --git a/tests/conformance/test_go_target.py b/tests/conformance/test_go_target.py new file mode 100644 index 0000000..353e6c4 --- /dev/null +++ b/tests/conformance/test_go_target.py @@ -0,0 +1,36 @@ +"""Go must pass the same conformance cases as Python. + +The test needs go. On a machine with no go it is skipped. +""" + +import json +import pathlib +import shutil +import sys + +import pytest + +CONFORMANCE = pathlib.Path(__file__).resolve().parents[2] / "conformance" +sys.path.insert(0, str(CONFORMANCE)) + +import run as conformance_run # noqa: E402 + + +@pytest.mark.skipif(shutil.which("go") is None, reason="needs go") +def test_go_passes_every_case(): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target("go:run", conformance_run.TARGETS["go:run"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + not_passed = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] != "pass"} + assert not_passed == {}, json.dumps(not_passed, indent=2) + + +def test_go_is_not_needed_for_the_default_targets(monkeypatch): + monkeypatch.setattr(conformance_run, "_has_go", lambda: False) + assert [name for name in conformance_run.default_targets() if name.startswith("go:")] == [] + + +def test_the_go_runner_cannot_download(): + env = conformance_run.TARGETS["go:run"]["env"] + assert env["GOPROXY"] == "off" and env["GOFLAGS"] == "-mod=mod" diff --git a/tests/conformance/test_judge_pipeline.py b/tests/conformance/test_judge_pipeline.py index 00fa78b..3570391 100644 --- a/tests/conformance/test_judge_pipeline.py +++ b/tests/conformance/test_judge_pipeline.py @@ -67,6 +67,14 @@ def test_a_rust_target_needs_cargo(): assert DefaultTargetsFilter(lambda: False, lambda: False).choose(targets, {}) == ["python:run"] +def test_a_go_target_needs_go(): + targets = {"python:run": {}, "rust:run": {}, "go:run": {}} + assert DefaultTargetsFilter(lambda: False, lambda: False, has_go=lambda: True).choose(targets, {}) == [ + "python:run", "go:run"] + assert DefaultTargetsFilter(lambda: False, lambda: True, has_go=lambda: False).choose(targets, {}) == [ + "python:run", "rust:run"] + + def test_two_cases_with_one_id_stop_the_run(tmp_path): (tmp_path / "a.json").write_text(json.dumps([{"id": "x"}, {"id": "x"}])) with pytest.raises(SystemExit, match="duplicate case ids"):